http.Client in goroutines

Question

I have a text file with this content:

192.168.1.2$nick
192.168.1.3$peter
192.168.1.4$mike
192.168.1.5$joe

A web server is running on each IP in the list.

I need to check if the servers are currently available and output a message if not available.

I wrote a small application. It works, but periodically produces incorrect results - it does not output messages for servers that are not actually available.

I can't figure out what's going on and in fact I'm not sure if I'm using http.Client correctly in goroutines.

Help me plese.

package main

import "fmt"
import "os"
import "strings"
import "io/ioutil"
import "net/http"
import "crypto/tls"
import "time"
import "strconv"

func makeGetRequest(URL string, c *http.Client) {
    resp, err := c.Get(URL)
    if err != nil {
        fmt.Println(err)
    }

    defer resp.Body.Close()

    if !((resp.StatusCode >= 200 && resp.StatusCode <= 209)) {
        fmt.Printf("%s-%d\n", URL, resp.StatusCode)
    }
}

func makeHeadRequestAsync(tasks chan string, done chan bool, c *http.Client) {

    for {
        URL := <-tasks

        if len(URL) == 0 {
            break
        }

        resp, err := c.Head(URL)
        if err != nil {
            fmt.Println(err)
            continue
        }
        defer resp.Body.Close() 
        if !((resp.StatusCode >= 200 && resp.StatusCode <= 209)) {
            makeGetRequest(URL, c) // !!! Some servers do not support HEAD requests. !!!
        }
    }

    done <- true
}

func main() {

    if len(os.Args) < 3 {
        fmt.Println("Usage: main <number of threads> <input-file>")
        os.Exit(0)
    }

    threadsNum, err := strconv.Atoi(os.Args[1]) 
    if err != nil {
        fmt.Println("Bad first parameter. Exit.")
        os.Exit(0)
    }

    http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
    client := &http.Client {
        Timeout: 30 * time.Second,
    }

    file, err := ioutil.ReadFile(os.Args[2])
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }

    fileLines := strings.Split(string(file), "\n") 

    tasks := make(chan string, threadsNum) 

    done := make(chan bool)

    for i := 0; i < threadsNum; i++ {
        go makeHeadRequestAsync(tasks, done, client)
    }

    for i := 0; i < len(fileLines); i++ {
        tasks <- strings.Split(string(fileLines[i]), "$")[0:1][0]
    }

    for i := 0; i < threadsNum; i++ {
        tasks <- ""
        <-done
    }
}

Answer 1

The program terminates when the main() function returns. The code does not ensure that all goroutines are done before returning from main.

Fix by doing the following:

• Use a sync.WaitGroup to wait for the goroutines to complete before exiting the program.
• Exit the goroutine when tasks is closed. Close tasks after submitting all work.

Here's the code:

func makeHeadRequestAsync(tasks chan string, wg *sync.WaitGroup, c *http.Client) {
    defer wg.Done()

    // for range on channel breaks when the channel is closed.
    for URL := range tasks {
        resp, err := c.Head(URL)
        if err != nil {
            fmt.Println(err)
            continue
        }
        defer resp.Body.Close()
        if !(resp.StatusCode >= 200 && resp.StatusCode <= 209) {
            makeGetRequest(URL, c) // !!! Some servers do not support HEAD requests. !!!
        }
    }

}

func main() {

    if len(os.Args) < 3 {
        fmt.Println("Usage: main <number of threads> <input-file>")
        os.Exit(0)
    }

    threadsNum, err := strconv.Atoi(os.Args[1])
    if err != nil {
        fmt.Println("Bad first parameter. Exit.")
        os.Exit(0)
    }

    http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
    client := &http.Client{
        Timeout: 30 * time.Second,
    }

    file, err := ioutil.ReadFile(os.Args[2])
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }

    fileLines := strings.Split(string(file), "\n")

    tasks := make(chan string)

    var wg sync.WaitGroup
    wg.Add(threadsNum)
    for i := 0; i < threadsNum; i++ {
        go makeHeadRequestAsync(tasks, &wg, client)
    }

    for i := 0; i < len(fileLines); i++ {
        tasks <- strings.Split(string(fileLines[i]), "$")[0:1][0]
    }
    close(tasks)
    wg.Wait()
}