How can I parse the standard Go package and print all constant variables?

51 Views Asked by At

Suppose I wish to extract and print all the constant variables defined within a specific version of the Go standard library. Below are the outlined steps:

  1. Download the source code of the Go standard library. For instance, 1.22.1.
  2. Copy the src directory to the location /foo/bar.
  3. For simplicity, exclude the /foo/bar/src/cmd directory.
  4. Utilize go/parser, go/types and go/ast to iterate through and print each constant definition.

Here's the code snippet I'm using:

package main

import (
    "cmp"
    "fmt"
    "go/ast"
    "go/build"
    "go/importer"
    "go/parser"
    "go/token"
    "go/types"
    "os"
    "path/filepath"
    "sort"
    "strings"
)

type PackageInfo struct {
    DirPath        string
    Name           string
    Files          map[string]*ast.File
    Fileset        *token.FileSet
    CheckedPackage *types.Package
}

func MapToSortedKeys[K cmp.Ordered, V any](m map[K]V) []K {
    keys := make([]K, len(m))
    i := 0
    for k := range m {
        keys[i] = k
        i++
    }
    sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] })
    return keys
}

func MapToValues[M ~map[K]V, K comparable, V any](m M) []V {
    r := make([]V, 0, len(m))
    for _, v := range m {
        r = append(r, v)
    }
    return r
}

func main() {
    errorsAll := []error{}

    rootSourceDirectory := "/Users/shu/Downloads/asdasd/src"
    dir2Files := map[string][]string{}
    filepath.Walk(rootSourceDirectory, func(path string, d os.FileInfo, err error) error {
        // Exclude non-go files and tests
        if strings.HasSuffix(path, ".go") && !strings.HasSuffix(path, "_test.go") {
            fileDir := filepath.Dir(path)

            // Exclude testdatas
            if !strings.Contains(fileDir, "/testdata/") {
                if _, ok := dir2Files[fileDir]; !ok {
                    dir2Files[fileDir] = []string{}
                }
                dir2Files[fileDir] = append(dir2Files[fileDir], path)
            }
        }
        return nil
    })

    buildContext := build.Default
    fset := token.NewFileSet()
    pkgDirPathToInfo := map[string]PackageInfo{}
    for _, fileDir := range MapToSortedKeys(dir2Files) {
        sourceFiles := dir2Files[fileDir]
        for _, sourceFile := range sourceFiles {
            if ok, err := buildContext.MatchFile(fileDir, filepath.Base(sourceFile)); err == nil && ok {
                fileDir := filepath.Dir(sourceFile)
                currFileSet := token.NewFileSet()
                file, err := parser.ParseFile(currFileSet, sourceFile, nil, parser.ParseComments)
                if err == nil {
                    if packageInfo, exists := pkgDirPathToInfo[fileDir]; exists {
                        packageInfo.Files[sourceFile] = file
                        currFileSet.Iterate(func(f *token.File) bool {
                            packageInfo.Fileset.AddFile(f.Name(), -1, f.Size())
                            fset.AddFile(f.Name(), -1, f.Size())
                            return true
                        })
                    } else {
                        pkgDirPathToInfo[fileDir] = PackageInfo{
                            DirPath:        fileDir,
                            Name:           file.Name.Name,
                            Files:          map[string]*ast.File{sourceFile: file},
                            Fileset:        currFileSet,
                            CheckedPackage: nil,
                        }
                    }
                } else {
                    fmt.Printf("file parsing failed for %s: %s\n", sourceFile, err)
                }
            }
        }
    }

    conf := types.Config{
        Importer:                 importer.ForCompiler(token.NewFileSet(), "source", nil),
        IgnoreFuncBodies:         true,
        FakeImportC:              true,
        DisableUnusedImportCheck: true,
        GoVersion:                "go1.22.1",
    }

    info := &types.Info{
        Types: make(map[ast.Expr]types.TypeAndValue),
    }
    for _, fileDir := range MapToSortedKeys(pkgDirPathToInfo) {
        pkgInfo := pkgDirPathToInfo[fileDir]
        checkedPackage, err := conf.Check(pkgInfo.Name, pkgInfo.Fileset, MapToValues(pkgInfo.Files), info)
        if err == nil {
            pkgInfo.CheckedPackage = checkedPackage
            pkgDirPathToInfo[fileDir] = pkgInfo
        } else {
            errorsAll = append(errorsAll, fmt.Errorf("%s: %s", pkgInfo.Name, err))
        }
    }

    fmt.Printf("Errors (%v):\n", len(errorsAll))
    for _, err := range errorsAll {
        fmt.Println(err)
    }

    fmt.Println()
    fmt.Printf("Constants (%v):\n", len(info.Types))
    for _, tv := range info.Types {
        if tv.Value != nil {
            fmt.Printf("%s: %s\n", tv.Type.String(), tv.Value.ExactString())
        }
    }
}

While this code generally works, I encounter errors related to dependencies, such as:

  • could not import golang.org/x/crypto/cryptobyte (no required module provides package golang.org/x/crypto/cryptobyte; to add it: go get golang.org/x/crypto/cryptobyte)
  • could not import github.com/mmcloughlin/avo/build (no required module provides package github.com/mmcloughlin/avo/build; to add it: go get github.com/mmcloughlin/avo/build)

The code works when I manually install the required dependency packages (as indicated in the errors). Nonetheless, this procedure must be repeated for all nested dependencies, the acquisition of which I'm uncertain about. Moreover, I must parse these packages during type checking, as certain constants may rely on constants defined within a dependency. Therefore, parsing the dependencies is essential to resolve the values accurately.

How can I retrieve all dependencies and configure my script to import them from a designated file path rather than depending on system-installed packages? One potential solution is to consolidate them within a directory, like /foo/bar/src/vendor.

0

There are 0 best solutions below