]>
Commit | Line | Data |
---|---|---|
bae9f6d2 JC |
1 | package module |
2 | ||
3 | import ( | |
4 | "bufio" | |
5 | "bytes" | |
6 | "fmt" | |
15c0b25d | 7 | "log" |
bae9f6d2 JC |
8 | "path/filepath" |
9 | "strings" | |
10 | "sync" | |
11 | ||
15c0b25d AP |
12 | "github.com/hashicorp/terraform/tfdiags" |
13 | ||
14 | getter "github.com/hashicorp/go-getter" | |
bae9f6d2 JC |
15 | "github.com/hashicorp/terraform/config" |
16 | ) | |
17 | ||
18 | // RootName is the name of the root tree. | |
19 | const RootName = "root" | |
20 | ||
21 | // Tree represents the module import tree of configurations. | |
22 | // | |
23 | // This Tree structure can be used to get (download) new modules, load | |
24 | // all the modules without getting, flatten the tree into something | |
25 | // Terraform can use, etc. | |
26 | type Tree struct { | |
27 | name string | |
28 | config *config.Config | |
29 | children map[string]*Tree | |
30 | path []string | |
31 | lock sync.RWMutex | |
15c0b25d AP |
32 | |
33 | // version is the final version of the config loaded for the Tree's module | |
34 | version string | |
35 | // source is the "source" string used to load this module. It's possible | |
36 | // for a module source to change, but the path remains the same, preventing | |
37 | // it from being reloaded. | |
38 | source string | |
39 | // parent allows us to walk back up the tree and determine if there are any | |
40 | // versioned ancestor modules which may effect the stored location of | |
41 | // submodules | |
42 | parent *Tree | |
bae9f6d2 JC |
43 | } |
44 | ||
45 | // NewTree returns a new Tree for the given config structure. | |
46 | func NewTree(name string, c *config.Config) *Tree { | |
47 | return &Tree{config: c, name: name} | |
48 | } | |
49 | ||
50 | // NewEmptyTree returns a new tree that is empty (contains no configuration). | |
51 | func NewEmptyTree() *Tree { | |
52 | t := &Tree{config: &config.Config{}} | |
53 | ||
54 | // We do this dummy load so that the tree is marked as "loaded". It | |
55 | // should never fail because this is just about a no-op. If it does fail | |
56 | // we panic so we can know its a bug. | |
15c0b25d | 57 | if err := t.Load(&Storage{Mode: GetModeGet}); err != nil { |
bae9f6d2 JC |
58 | panic(err) |
59 | } | |
60 | ||
61 | return t | |
62 | } | |
63 | ||
64 | // NewTreeModule is like NewTree except it parses the configuration in | |
65 | // the directory and gives it a specific name. Use a blank name "" to specify | |
66 | // the root module. | |
67 | func NewTreeModule(name, dir string) (*Tree, error) { | |
68 | c, err := config.LoadDir(dir) | |
69 | if err != nil { | |
70 | return nil, err | |
71 | } | |
72 | ||
73 | return NewTree(name, c), nil | |
74 | } | |
75 | ||
76 | // Config returns the configuration for this module. | |
77 | func (t *Tree) Config() *config.Config { | |
78 | return t.config | |
79 | } | |
80 | ||
81 | // Child returns the child with the given path (by name). | |
82 | func (t *Tree) Child(path []string) *Tree { | |
83 | if t == nil { | |
84 | return nil | |
85 | } | |
86 | ||
87 | if len(path) == 0 { | |
88 | return t | |
89 | } | |
90 | ||
91 | c := t.Children()[path[0]] | |
92 | if c == nil { | |
93 | return nil | |
94 | } | |
95 | ||
96 | return c.Child(path[1:]) | |
97 | } | |
98 | ||
99 | // Children returns the children of this tree (the modules that are | |
100 | // imported by this root). | |
101 | // | |
102 | // This will only return a non-nil value after Load is called. | |
103 | func (t *Tree) Children() map[string]*Tree { | |
104 | t.lock.RLock() | |
105 | defer t.lock.RUnlock() | |
106 | return t.children | |
107 | } | |
108 | ||
c680a8e1 RS |
109 | // DeepEach calls the provided callback for the receiver and then all of |
110 | // its descendents in the tree, allowing an operation to be performed on | |
111 | // all modules in the tree. | |
112 | // | |
113 | // Parents will be visited before their children but otherwise the order is | |
114 | // not defined. | |
115 | func (t *Tree) DeepEach(cb func(*Tree)) { | |
116 | t.lock.RLock() | |
117 | defer t.lock.RUnlock() | |
118 | t.deepEach(cb) | |
119 | } | |
120 | ||
121 | func (t *Tree) deepEach(cb func(*Tree)) { | |
122 | cb(t) | |
123 | for _, c := range t.children { | |
124 | c.deepEach(cb) | |
125 | } | |
126 | } | |
127 | ||
bae9f6d2 JC |
128 | // Loaded says whether or not this tree has been loaded or not yet. |
129 | func (t *Tree) Loaded() bool { | |
130 | t.lock.RLock() | |
131 | defer t.lock.RUnlock() | |
132 | return t.children != nil | |
133 | } | |
134 | ||
135 | // Modules returns the list of modules that this tree imports. | |
136 | // | |
137 | // This is only the imports of _this_ level of the tree. To retrieve the | |
138 | // full nested imports, you'll have to traverse the tree. | |
139 | func (t *Tree) Modules() []*Module { | |
140 | result := make([]*Module, len(t.config.Modules)) | |
141 | for i, m := range t.config.Modules { | |
142 | result[i] = &Module{ | |
15c0b25d AP |
143 | Name: m.Name, |
144 | Version: m.Version, | |
145 | Source: m.Source, | |
146 | Providers: m.Providers, | |
bae9f6d2 JC |
147 | } |
148 | } | |
149 | ||
150 | return result | |
151 | } | |
152 | ||
153 | // Name returns the name of the tree. This will be "<root>" for the root | |
154 | // tree and then the module name given for any children. | |
155 | func (t *Tree) Name() string { | |
156 | if t.name == "" { | |
157 | return RootName | |
158 | } | |
159 | ||
160 | return t.name | |
161 | } | |
162 | ||
163 | // Load loads the configuration of the entire tree. | |
164 | // | |
165 | // The parameters are used to tell the tree where to find modules and | |
166 | // whether it can download/update modules along the way. | |
167 | // | |
168 | // Calling this multiple times will reload the tree. | |
169 | // | |
170 | // Various semantic-like checks are made along the way of loading since | |
171 | // module trees inherently require the configuration to be in a reasonably | |
172 | // sane state: no circular dependencies, proper module sources, etc. A full | |
173 | // suite of validations can be done by running Validate (after loading). | |
15c0b25d | 174 | func (t *Tree) Load(s *Storage) error { |
bae9f6d2 JC |
175 | t.lock.Lock() |
176 | defer t.lock.Unlock() | |
177 | ||
15c0b25d AP |
178 | children, err := t.getChildren(s) |
179 | if err != nil { | |
180 | return err | |
181 | } | |
182 | ||
183 | // Go through all the children and load them. | |
184 | for _, c := range children { | |
185 | if err := c.Load(s); err != nil { | |
186 | return err | |
187 | } | |
188 | } | |
189 | ||
190 | // Set our tree up | |
191 | t.children = children | |
bae9f6d2 | 192 | |
15c0b25d AP |
193 | return nil |
194 | } | |
195 | ||
196 | func (t *Tree) getChildren(s *Storage) (map[string]*Tree, error) { | |
bae9f6d2 JC |
197 | children := make(map[string]*Tree) |
198 | ||
199 | // Go through all the modules and get the directory for them. | |
15c0b25d | 200 | for _, m := range t.Modules() { |
bae9f6d2 | 201 | if _, ok := children[m.Name]; ok { |
15c0b25d | 202 | return nil, fmt.Errorf( |
bae9f6d2 JC |
203 | "module %s: duplicated. module names must be unique", m.Name) |
204 | } | |
205 | ||
206 | // Determine the path to this child | |
15c0b25d AP |
207 | modPath := make([]string, len(t.path), len(t.path)+1) |
208 | copy(modPath, t.path) | |
209 | modPath = append(modPath, m.Name) | |
bae9f6d2 | 210 | |
15c0b25d | 211 | log.Printf("[TRACE] module source: %q", m.Source) |
bae9f6d2 | 212 | |
15c0b25d AP |
213 | // add the module path to help indicate where modules with relative |
214 | // paths are being loaded from | |
215 | s.output(fmt.Sprintf("- module.%s", strings.Join(modPath, "."))) | |
216 | ||
217 | // Lookup the local location of the module. | |
218 | // dir is the local directory where the module is stored | |
219 | mod, err := s.findRegistryModule(m.Source, m.Version) | |
bae9f6d2 | 220 | if err != nil { |
15c0b25d | 221 | return nil, err |
bae9f6d2 JC |
222 | } |
223 | ||
15c0b25d AP |
224 | // The key is the string that will be used to uniquely id the Source in |
225 | // the local storage. The prefix digit can be incremented to | |
226 | // invalidate the local module storage. | |
227 | key := "1." + t.versionedPathKey(m) | |
228 | if mod.Version != "" { | |
229 | key += "." + mod.Version | |
230 | } | |
231 | ||
232 | // Check for the exact key if it's not a registry module | |
233 | if !mod.registry { | |
234 | mod.Dir, err = s.findModule(key) | |
235 | if err != nil { | |
236 | return nil, err | |
237 | } | |
238 | } | |
239 | ||
240 | if mod.Dir != "" && s.Mode != GetModeUpdate { | |
241 | // We found it locally, but in order to load the Tree we need to | |
242 | // find out if there was another subDir stored from detection. | |
243 | subDir, err := s.getModuleRoot(mod.Dir) | |
244 | if err != nil { | |
245 | // If there's a problem with the subdir record, we'll let the | |
246 | // recordSubdir method fix it up. Any other filesystem errors | |
247 | // will turn up again below. | |
248 | log.Println("[WARN] error reading subdir record:", err) | |
249 | } | |
250 | ||
251 | fullDir := filepath.Join(mod.Dir, subDir) | |
252 | ||
253 | child, err := NewTreeModule(m.Name, fullDir) | |
254 | if err != nil { | |
255 | return nil, fmt.Errorf("module %s: %s", m.Name, err) | |
256 | } | |
257 | child.path = modPath | |
258 | child.parent = t | |
259 | child.version = mod.Version | |
260 | child.source = m.Source | |
261 | children[m.Name] = child | |
262 | continue | |
263 | } | |
264 | ||
265 | // Split out the subdir if we have one. | |
266 | // Terraform keeps the entire requested tree, so that modules can | |
267 | // reference sibling modules from the same archive or repo. | |
268 | rawSource, subDir := getter.SourceDirSubdir(m.Source) | |
269 | ||
270 | // we haven't found a source, so fallback to the go-getter detectors | |
271 | source := mod.url | |
272 | if source == "" { | |
273 | source, err = getter.Detect(rawSource, t.config.Dir, getter.Detectors) | |
274 | if err != nil { | |
275 | return nil, fmt.Errorf("module %s: %s", m.Name, err) | |
276 | } | |
277 | } | |
278 | ||
279 | log.Printf("[TRACE] detected module source %q", source) | |
280 | ||
bae9f6d2 | 281 | // Check if the detector introduced something new. |
15c0b25d AP |
282 | // For example, the registry always adds a subdir of `//*`, |
283 | // indicating that we need to strip off the first component from the | |
284 | // tar archive, though we may not yet know what it is called. | |
285 | source, detectedSubDir := getter.SourceDirSubdir(source) | |
286 | if detectedSubDir != "" { | |
287 | subDir = filepath.Join(detectedSubDir, subDir) | |
288 | } | |
289 | ||
290 | output := "" | |
291 | switch s.Mode { | |
292 | case GetModeUpdate: | |
293 | output = fmt.Sprintf(" Updating source %q", m.Source) | |
294 | default: | |
295 | output = fmt.Sprintf(" Getting source %q", m.Source) | |
bae9f6d2 | 296 | } |
15c0b25d | 297 | s.output(output) |
bae9f6d2 | 298 | |
15c0b25d | 299 | dir, ok, err := s.getStorage(key, source) |
bae9f6d2 | 300 | if err != nil { |
15c0b25d | 301 | return nil, err |
bae9f6d2 JC |
302 | } |
303 | if !ok { | |
15c0b25d | 304 | return nil, fmt.Errorf("module %s: not found, may need to run 'terraform init'", m.Name) |
bae9f6d2 JC |
305 | } |
306 | ||
15c0b25d AP |
307 | log.Printf("[TRACE] %q stored in %q", source, dir) |
308 | ||
309 | // expand and record the subDir for later | |
310 | fullDir := dir | |
bae9f6d2 | 311 | if subDir != "" { |
15c0b25d AP |
312 | fullDir, err = getter.SubdirGlob(dir, subDir) |
313 | if err != nil { | |
314 | return nil, err | |
315 | } | |
bae9f6d2 | 316 | |
15c0b25d AP |
317 | // +1 to account for the pathsep |
318 | if len(dir)+1 > len(fullDir) { | |
319 | return nil, fmt.Errorf("invalid module storage path %q", fullDir) | |
320 | } | |
321 | subDir = fullDir[len(dir)+1:] | |
bae9f6d2 JC |
322 | } |
323 | ||
15c0b25d AP |
324 | // add new info to the module record |
325 | mod.Key = key | |
326 | mod.Dir = dir | |
327 | mod.Root = subDir | |
bae9f6d2 | 328 | |
15c0b25d AP |
329 | // record the module in our manifest |
330 | if err := s.recordModule(mod); err != nil { | |
331 | return nil, err | |
bae9f6d2 | 332 | } |
bae9f6d2 | 333 | |
15c0b25d AP |
334 | child, err := NewTreeModule(m.Name, fullDir) |
335 | if err != nil { | |
336 | return nil, fmt.Errorf("module %s: %s", m.Name, err) | |
337 | } | |
338 | child.path = modPath | |
339 | child.parent = t | |
340 | child.version = mod.Version | |
341 | child.source = m.Source | |
342 | children[m.Name] = child | |
343 | } | |
bae9f6d2 | 344 | |
15c0b25d | 345 | return children, nil |
bae9f6d2 JC |
346 | } |
347 | ||
348 | // Path is the full path to this tree. | |
349 | func (t *Tree) Path() []string { | |
350 | return t.path | |
351 | } | |
352 | ||
353 | // String gives a nice output to describe the tree. | |
354 | func (t *Tree) String() string { | |
355 | var result bytes.Buffer | |
356 | path := strings.Join(t.path, ", ") | |
357 | if path != "" { | |
358 | path = fmt.Sprintf(" (path: %s)", path) | |
359 | } | |
360 | result.WriteString(t.Name() + path + "\n") | |
361 | ||
362 | cs := t.Children() | |
363 | if cs == nil { | |
364 | result.WriteString(" not loaded") | |
365 | } else { | |
366 | // Go through each child and get its string value, then indent it | |
367 | // by two. | |
368 | for _, c := range cs { | |
369 | r := strings.NewReader(c.String()) | |
370 | scanner := bufio.NewScanner(r) | |
371 | for scanner.Scan() { | |
372 | result.WriteString(" ") | |
373 | result.WriteString(scanner.Text()) | |
374 | result.WriteString("\n") | |
375 | } | |
376 | } | |
377 | } | |
378 | ||
379 | return result.String() | |
380 | } | |
381 | ||
382 | // Validate does semantic checks on the entire tree of configurations. | |
383 | // | |
384 | // This will call the respective config.Config.Validate() functions as well | |
385 | // as verifying things such as parameters/outputs between the various modules. | |
386 | // | |
387 | // Load must be called prior to calling Validate or an error will be returned. | |
15c0b25d AP |
388 | func (t *Tree) Validate() tfdiags.Diagnostics { |
389 | var diags tfdiags.Diagnostics | |
390 | ||
bae9f6d2 | 391 | if !t.Loaded() { |
15c0b25d AP |
392 | diags = diags.Append(fmt.Errorf( |
393 | "tree must be loaded before calling Validate", | |
394 | )) | |
395 | return diags | |
bae9f6d2 JC |
396 | } |
397 | ||
bae9f6d2 JC |
398 | // Terraform core does not handle root module children named "root". |
399 | // We plan to fix this in the future but this bug was brought up in | |
400 | // the middle of a release and we don't want to introduce wide-sweeping | |
401 | // changes at that time. | |
402 | if len(t.path) == 1 && t.name == "root" { | |
15c0b25d AP |
403 | diags = diags.Append(fmt.Errorf( |
404 | "root module cannot contain module named 'root'", | |
405 | )) | |
406 | return diags | |
bae9f6d2 JC |
407 | } |
408 | ||
409 | // Validate our configuration first. | |
15c0b25d | 410 | diags = diags.Append(t.config.Validate()) |
bae9f6d2 JC |
411 | |
412 | // If we're the root, we do extra validation. This validation usually | |
413 | // requires the entire tree (since children don't have parent pointers). | |
414 | if len(t.path) == 0 { | |
415 | if err := t.validateProviderAlias(); err != nil { | |
15c0b25d | 416 | diags = diags.Append(err) |
bae9f6d2 JC |
417 | } |
418 | } | |
419 | ||
420 | // Get the child trees | |
421 | children := t.Children() | |
422 | ||
423 | // Validate all our children | |
424 | for _, c := range children { | |
15c0b25d AP |
425 | childDiags := c.Validate() |
426 | diags = diags.Append(childDiags) | |
427 | if diags.HasErrors() { | |
bae9f6d2 JC |
428 | continue |
429 | } | |
bae9f6d2 JC |
430 | } |
431 | ||
432 | // Go over all the modules and verify that any parameters are valid | |
433 | // variables into the module in question. | |
434 | for _, m := range t.config.Modules { | |
435 | tree, ok := children[m.Name] | |
436 | if !ok { | |
437 | // This should never happen because Load watches us | |
438 | panic("module not found in children: " + m.Name) | |
439 | } | |
440 | ||
441 | // Build the variables that the module defines | |
442 | requiredMap := make(map[string]struct{}) | |
443 | varMap := make(map[string]struct{}) | |
444 | for _, v := range tree.config.Variables { | |
445 | varMap[v.Name] = struct{}{} | |
446 | ||
447 | if v.Required() { | |
448 | requiredMap[v.Name] = struct{}{} | |
449 | } | |
450 | } | |
451 | ||
452 | // Compare to the keys in our raw config for the module | |
453 | for k, _ := range m.RawConfig.Raw { | |
454 | if _, ok := varMap[k]; !ok { | |
15c0b25d AP |
455 | diags = diags.Append(fmt.Errorf( |
456 | "module %q: %q is not a valid argument", | |
457 | m.Name, k, | |
458 | )) | |
bae9f6d2 JC |
459 | } |
460 | ||
461 | // Remove the required | |
462 | delete(requiredMap, k) | |
463 | } | |
464 | ||
465 | // If we have any required left over, they aren't set. | |
466 | for k, _ := range requiredMap { | |
15c0b25d AP |
467 | diags = diags.Append(fmt.Errorf( |
468 | "module %q: missing required argument %q", | |
469 | m.Name, k, | |
470 | )) | |
bae9f6d2 JC |
471 | } |
472 | } | |
473 | ||
474 | // Go over all the variables used and make sure that any module | |
475 | // variables represent outputs properly. | |
476 | for source, vs := range t.config.InterpolatedVariables() { | |
477 | for _, v := range vs { | |
478 | mv, ok := v.(*config.ModuleVariable) | |
479 | if !ok { | |
480 | continue | |
481 | } | |
482 | ||
483 | tree, ok := children[mv.Name] | |
484 | if !ok { | |
15c0b25d AP |
485 | diags = diags.Append(fmt.Errorf( |
486 | "%s: reference to undefined module %q", | |
487 | source, mv.Name, | |
488 | )) | |
bae9f6d2 JC |
489 | continue |
490 | } | |
491 | ||
492 | found := false | |
493 | for _, o := range tree.config.Outputs { | |
494 | if o.Name == mv.Field { | |
495 | found = true | |
496 | break | |
497 | } | |
498 | } | |
499 | if !found { | |
15c0b25d AP |
500 | diags = diags.Append(fmt.Errorf( |
501 | "%s: %q is not a valid output for module %q", | |
502 | source, mv.Field, mv.Name, | |
503 | )) | |
bae9f6d2 JC |
504 | } |
505 | } | |
506 | } | |
507 | ||
15c0b25d AP |
508 | return diags |
509 | } | |
510 | ||
511 | // versionedPathKey returns a path string with every levels full name, version | |
512 | // and source encoded. This is to provide a unique key for our module storage, | |
513 | // since submodules need to know which versions of their ancestor modules they | |
514 | // are loaded from. | |
515 | // For example, if module A has a subdirectory B, if module A's source or | |
516 | // version is updated B's storage key must reflect this change in order for the | |
517 | // correct version of B's source to be loaded. | |
518 | func (t *Tree) versionedPathKey(m *Module) string { | |
519 | path := make([]string, len(t.path)+1) | |
520 | path[len(path)-1] = m.Name + ";" + m.Source | |
521 | // We're going to load these in order for easier reading and debugging, but | |
522 | // in practice they only need to be unique and consistent. | |
523 | ||
524 | p := t | |
525 | i := len(path) - 2 | |
526 | for ; i >= 0; i-- { | |
527 | if p == nil { | |
528 | break | |
529 | } | |
530 | // we may have been loaded under a blank Tree, so always check for a name | |
531 | // too. | |
532 | if p.name == "" { | |
533 | break | |
534 | } | |
535 | seg := p.name | |
536 | if p.version != "" { | |
537 | seg += "#" + p.version | |
538 | } | |
539 | ||
540 | if p.source != "" { | |
541 | seg += ";" + p.source | |
542 | } | |
543 | ||
544 | path[i] = seg | |
545 | p = p.parent | |
546 | } | |
547 | ||
548 | key := strings.Join(path, "|") | |
549 | return key | |
bae9f6d2 JC |
550 | } |
551 | ||
552 | // treeError is an error use by Tree.Validate to accumulates all | |
553 | // validation errors. | |
554 | type treeError struct { | |
555 | Name []string | |
556 | Errs []error | |
557 | Children []*treeError | |
558 | } | |
559 | ||
560 | func (e *treeError) Add(err error) { | |
561 | e.Errs = append(e.Errs, err) | |
562 | } | |
563 | ||
564 | func (e *treeError) AddChild(err *treeError) { | |
565 | e.Children = append(e.Children, err) | |
566 | } | |
567 | ||
568 | func (e *treeError) ErrOrNil() error { | |
569 | if len(e.Errs) > 0 || len(e.Children) > 0 { | |
570 | return e | |
571 | } | |
572 | return nil | |
573 | } | |
574 | ||
575 | func (e *treeError) Error() string { | |
576 | name := strings.Join(e.Name, ".") | |
577 | var out bytes.Buffer | |
578 | fmt.Fprintf(&out, "module %s: ", name) | |
579 | ||
580 | if len(e.Errs) == 1 { | |
581 | // single like error | |
582 | out.WriteString(e.Errs[0].Error()) | |
583 | } else { | |
584 | // multi-line error | |
585 | for _, err := range e.Errs { | |
586 | fmt.Fprintf(&out, "\n %s", err) | |
587 | } | |
588 | } | |
589 | ||
590 | if len(e.Children) > 0 { | |
591 | // start the next error on a new line | |
592 | out.WriteString("\n ") | |
593 | } | |
594 | for _, child := range e.Children { | |
595 | out.WriteString(child.Error()) | |
596 | } | |
597 | ||
598 | return out.String() | |
599 | } |