Preload providers

.gitignore

@@ -1,3 +1,7 @@
 dist/
 .idea/
 coverage.txt
+internal/schemas/.terraform/
+internal/schemas/dev.log
+internal/schemas/providers.tf
+internal/schemas/data/

go.mod

@@ -20,6 +20,8 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.3.2
 	github.com/pmezard/go-difflib v1.0.0
+	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
+	github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546
 	github.com/sourcegraph/go-lsp v0.0.0-20200117082640-b19bb38222e2
 	github.com/spf13/afero v1.3.2
 	github.com/stretchr/testify v1.4.0

go.sum

@@ -315,7 +315,11 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk=
+github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 h1:pXY9qYc/MP5zdvqWEUH6SjNiu7VhSjuVFTFiTcphaLU=
+github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=

internal/schemas/doc.go

@@ -0,0 +1,5 @@
+// Package schemas embeds provder schemas for completion support, in particular,
+// for uninitialized modules
+package schemas
+
+//go:generate go run gen/gen.go

internal/schemas/gen/gen.go

@@ -0,0 +1,173 @@
+// +build generate
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/hashicorp/terraform-exec/tfexec"
+	"github.com/hashicorp/terraform-exec/tfinstall"
+	"github.com/shurcooL/vfsgen"
+)
+
+const terraformBlock = `terraform {
+  required_providers {
+  {{ range $p := . }}
+    {{ $p.Name }} = {
+      source = "{{ $p.Source }}"
+    }
+  {{ end }}
+  }
+}
+`
+
+func main() {
+	os.Exit(func() int {
+		if err := gen(); err != nil {
+			fmt.Fprintf(os.Stderr, "error: %v\n", err)
+			return 1
+		}
+		return 0
+	}())
+}
+
+func gen() error {
+	ctx := context.Background()
+
+	log.Println("fetching official providers from registry")
+	providers, err := listProviders("official")
+	if err != nil {
+		return err
+	}
+
+	log.Println("parsing template")
+	tmpl, err := template.New("providers").Parse(terraformBlock)
+	if err != nil {
+		return err
+	}
+
+	log.Println("creating config file")
+	configFile, err := os.Create("providers.tf")
+	if err != nil {
+		return err
+	}
+
+	log.Println("executing template")
+	err = tmpl.Execute(configFile, providers)
+	if err != nil {
+		return err
+	}
+
+	log.Println("finding terraform in path")
+	execPath, err := tfinstall.Find(ctx, tfinstall.LookPath())
+	if err != nil {
+		log.Println("terraform not found in path, attempting install to temp")
+
+		tmpDir, err := ioutil.TempDir("", "tfinstall")
+		if err != nil {
+			return err
+		}
+		defer os.RemoveAll(tmpDir)
+
+		execPath, err = tfinstall.Find(ctx, tfinstall.LatestVersion(tmpDir, false))
+		if err != nil {
+			return err
+		}
+	}
+
+	log.Println("running terraform init")
+
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	tf, err := tfexec.NewTerraform(cwd, execPath)
+	if err != nil {
+		return err
+	}
+
+	err = tf.Init(ctx, tfexec.Upgrade(true), tfexec.LockTimeout("120s"))
+	if err != nil {
+		return err
+	}
+
+	// TODO upstream change to have tfexec write to file directly instead of unmarshal/remarshal
+	log.Println("running terraform providers schema")
+	ps, err := tf.ProvidersSchema(ctx)
+	if err != nil {
+		return err
+	}
+
+	log.Println("creating schemas/data dir")
+	err = os.MkdirAll("data", 0755)
+	if err != nil {
+		return err
+	}
+
+	log.Println("creating schemas file")
+	schemasFile, err := os.Create(filepath.Join("data", "schemas.json"))
+	if err != nil {
+		return err
+	}
+
+	log.Println("writing schemas to file")
+	err = json.NewEncoder(schemasFile).Encode(ps)
+	if err != nil {
+		return err
+	}
+
+	log.Println("generating embedded go file")
+	var fs http.FileSystem = http.Dir("data")
+	return vfsgen.Generate(fs, vfsgen.Options{
+		Filename:     "schemas.go",
+		PackageName:  "schemas",
+		VariableName: "Files",
+	})
+}
+
+type providerAttributes struct {
+	Alias    string `json:"alias"`
+	FullName string `json:"full-name"`
+}
+
+type provider struct {
+	Attributes providerAttributes `json:"attributes"`
+}
+
+func (p provider) Name() string {
+	return p.Attributes.Alias
+}
+
+func (p provider) Source() string {
+	// terraform provider is builtin and has special source
+	if p.Attributes.Alias == "terraform" {
+		return "terraform.io/builtin/terraform"
+	}
+	return p.Attributes.FullName
+}
+
+type registryResponse struct {
+	Data []provider `json:"data"`
+}
+
+func listProviders(tier string) ([]provider, error) {
+	// TODO will eventually need to paginate, for now "official" is 33 and "partner" is 82
+	resp, err := http.Get(fmt.Sprintf("https://registry.terraform.io/v2/providers?page[size]=100&filter[tier]=%s", tier))
+	if err != nil {
+		return nil, err
+	}
+
+	var response registryResponse
+	err = json.NewDecoder(resp.Body).Decode(&response)
+
+	return response.Data, err
+}

internal/terraform/rootmodule/root_module.go

@@ -2,6 +2,7 @@ package rootmodule
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -20,11 +21,16 @@ import (
 	"github.com/hashicorp/hcl/v2/hclsyntax"
 	tfjson "github.com/hashicorp/terraform-json"
 	"github.com/hashicorp/terraform-ls/internal/filesystem"
+	preloadedSchemas "github.com/hashicorp/terraform-ls/internal/schemas"
 	"github.com/hashicorp/terraform-ls/internal/terraform/discovery"
 	"github.com/hashicorp/terraform-ls/internal/terraform/exec"
 	tfschema "github.com/hashicorp/terraform-schema/schema"
 )
 
+var _preloadedProviderSchemas *tfjson.ProviderSchemas
+var _preloadedProviderSchemasOnce sync.Once
+var _preloadedProviderSchemasErr error
+
 type rootModule struct {
 	path   string
 	logger *log.Logger
@@ -76,6 +82,21 @@ type rootModule struct {
 	filesystem  filesystem.Filesystem
 }
 
+func preloadedProviderSchemas() (*tfjson.ProviderSchemas, error) {
+	_preloadedProviderSchemasOnce.Do(func() {
+		f, fErr := preloadedSchemas.Files.Open("schemas.json")
+		if fErr != nil {
+			_preloadedProviderSchemasErr = fErr
+			return
+		}
+
+		_preloadedProviderSchemas = &tfjson.ProviderSchemas{}
+		_preloadedProviderSchemasErr = json.NewDecoder(f).Decode(_preloadedProviderSchemas)
+	})
+
+	return _preloadedProviderSchemas, _preloadedProviderSchemasErr
+}
+
 func newRootModule(fs filesystem.Filesystem, dir string) *rootModule {
 	return &rootModule{
 		path:             dir,
@@ -478,24 +499,34 @@ func (rm *rootModule) parsedFiles() map[string]*hcl.File {
 }
 
 func (rm *rootModule) MergedSchema() (*schema.BodySchema, error) {
+
 	rm.coreSchemaMu.RLock()
 	defer rm.coreSchemaMu.RUnlock()
 	mergedSchema := rm.coreSchema
 
-	if rm.IsProviderSchemaLoaded() {
-		if !rm.IsParsed() {
-			err := rm.ParseFiles()
-			if err != nil {
-				return nil, err
-			}
-		}
-		s, err := tfschema.MergeCoreWithJsonProviderSchemas(rm.parsedFiles(), mergedSchema, rm.providerSchema)
+	if !rm.IsParsed() {
+		err := rm.ParseFiles()
 		if err != nil {
 			return nil, err
 		}
-		mergedSchema = s
 	}
 
+	ps, err := preloadedProviderSchemas()
+	if err != nil {
+		return nil, err
+	}
+	if rm.IsProviderSchemaLoaded() {
+		rm.providerSchemaMu.RLock()
+		defer rm.providerSchemaMu.RUnlock()
+		ps = rm.providerSchema
+	}
+
+	s, err := tfschema.MergeCoreWithJsonProviderSchemas(rm.parsedFiles(), mergedSchema, ps)
+	if err != nil {
+		return nil, err
+	}
+	mergedSchema = s
+
 	return mergedSchema, nil
 }