mirror of
https://github.com/opentofu/opentofu.git
synced 2025-01-20 21:52:57 -06:00
ef4726bd50
Changing the Set internals makes a lot of sense as it saves doing conversions in multiple places and gives a central place to alter the key when a item is computed. This will have no side effects other then that the ordering is now based on strings instead on integers, so the order will be different. This will however have no effect on existing configs as these will use the individual codes/keys and not the ordering to determine if there is a diff or not. Lastly (but I think also most importantly) there is a fix in this PR that makes diffing sets extremely more performand. Before a full diff required reading the complete Set for every single parameter/attribute you wanted to diff, while now it only gets that specific parameter. We have a use case where we have a Set that has 18 parameters and the set consist of about 600 items (don't ask 😉). So when doing a diff it would take 100% CPU of all cores and stay that way for almost an hour before being able to complete the diff. Debugging this we learned that for retrieving every single parameter it made over 52.000 calls to `func (c *ResourceConfig) get(..)`. In this function a slice is created and used only for the duration of the call, so the time needed to create all needed slices and on the other hand the time the garbage collector needed to clean them up again caused the system to cripple itself. Next to that there are also some expensive reflect calls in this function which also claimed a fair amount of CPU time. After this fix the number of calls needed to get a single parameter dropped from 52.000+ to only 2! 😃
273 lines
6.7 KiB
Go
273 lines
6.7 KiB
Go
package schema
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
|
|
"github.com/hashicorp/terraform/terraform"
|
|
"github.com/mitchellh/mapstructure"
|
|
)
|
|
|
|
// ConfigFieldReader reads fields out of an untyped map[string]string to the
|
|
// best of its ability. It also applies defaults from the Schema. (The other
|
|
// field readers do not need default handling because they source fully
|
|
// populated data structures.)
|
|
type ConfigFieldReader struct {
|
|
Config *terraform.ResourceConfig
|
|
Schema map[string]*Schema
|
|
|
|
indexMaps map[string]map[string]int
|
|
once sync.Once
|
|
}
|
|
|
|
func (r *ConfigFieldReader) ReadField(address []string) (FieldReadResult, error) {
|
|
r.once.Do(func() { r.indexMaps = make(map[string]map[string]int) })
|
|
return r.readField(address, false)
|
|
}
|
|
|
|
func (r *ConfigFieldReader) readField(
|
|
address []string, nested bool) (FieldReadResult, error) {
|
|
schemaList := addrToSchema(address, r.Schema)
|
|
if len(schemaList) == 0 {
|
|
return FieldReadResult{}, nil
|
|
}
|
|
|
|
if !nested {
|
|
// If we have a set anywhere in the address, then we need to
|
|
// read that set out in order and actually replace that part of
|
|
// the address with the real list index. i.e. set.50 might actually
|
|
// map to set.12 in the config, since it is in list order in the
|
|
// config, not indexed by set value.
|
|
for i, v := range schemaList {
|
|
// Sets are the only thing that cause this issue.
|
|
if v.Type != TypeSet {
|
|
continue
|
|
}
|
|
|
|
// If we're at the end of the list, then we don't have to worry
|
|
// about this because we're just requesting the whole set.
|
|
if i == len(schemaList)-1 {
|
|
continue
|
|
}
|
|
|
|
// If we're looking for the count, then ignore...
|
|
if address[i+1] == "#" {
|
|
continue
|
|
}
|
|
|
|
indexMap, ok := r.indexMaps[strings.Join(address[:i+1], ".")]
|
|
if !ok {
|
|
// Get the set so we can get the index map that tells us the
|
|
// mapping of the hash code to the list index
|
|
_, err := r.readSet(address[:i+1], v)
|
|
if err != nil {
|
|
return FieldReadResult{}, err
|
|
}
|
|
indexMap = r.indexMaps[strings.Join(address[:i+1], ".")]
|
|
}
|
|
|
|
index, ok := indexMap[address[i+1]]
|
|
if !ok {
|
|
return FieldReadResult{}, nil
|
|
}
|
|
|
|
address[i+1] = strconv.FormatInt(int64(index), 10)
|
|
}
|
|
}
|
|
|
|
k := strings.Join(address, ".")
|
|
schema := schemaList[len(schemaList)-1]
|
|
switch schema.Type {
|
|
case TypeBool, TypeFloat, TypeInt, TypeString:
|
|
return r.readPrimitive(k, schema)
|
|
case TypeList:
|
|
return readListField(&nestedConfigFieldReader{r}, address, schema)
|
|
case TypeMap:
|
|
return r.readMap(k)
|
|
case TypeSet:
|
|
return r.readSet(address, schema)
|
|
case typeObject:
|
|
return readObjectField(
|
|
&nestedConfigFieldReader{r},
|
|
address, schema.Elem.(map[string]*Schema))
|
|
default:
|
|
panic(fmt.Sprintf("Unknown type: %s", schema.Type))
|
|
}
|
|
}
|
|
|
|
func (r *ConfigFieldReader) readMap(k string) (FieldReadResult, error) {
|
|
// We want both the raw value and the interpolated. We use the interpolated
|
|
// to store actual values and we use the raw one to check for
|
|
// computed keys.
|
|
mraw, ok := r.Config.GetRaw(k)
|
|
if !ok {
|
|
return FieldReadResult{}, nil
|
|
}
|
|
|
|
result := make(map[string]interface{})
|
|
computed := false
|
|
switch m := mraw.(type) {
|
|
case []interface{}:
|
|
for i, innerRaw := range m {
|
|
for ik := range innerRaw.(map[string]interface{}) {
|
|
key := fmt.Sprintf("%s.%d.%s", k, i, ik)
|
|
if r.Config.IsComputed(key) {
|
|
computed = true
|
|
break
|
|
}
|
|
|
|
v, _ := r.Config.Get(key)
|
|
result[ik] = v
|
|
}
|
|
}
|
|
case []map[string]interface{}:
|
|
for i, innerRaw := range m {
|
|
for ik := range innerRaw {
|
|
key := fmt.Sprintf("%s.%d.%s", k, i, ik)
|
|
if r.Config.IsComputed(key) {
|
|
computed = true
|
|
break
|
|
}
|
|
|
|
v, _ := r.Config.Get(key)
|
|
result[ik] = v
|
|
}
|
|
}
|
|
case map[string]interface{}:
|
|
for ik := range m {
|
|
key := fmt.Sprintf("%s.%s", k, ik)
|
|
if r.Config.IsComputed(key) {
|
|
computed = true
|
|
break
|
|
}
|
|
|
|
v, _ := r.Config.Get(key)
|
|
result[ik] = v
|
|
}
|
|
default:
|
|
panic(fmt.Sprintf("unknown type: %#v", mraw))
|
|
}
|
|
|
|
var value interface{}
|
|
if !computed {
|
|
value = result
|
|
}
|
|
|
|
return FieldReadResult{
|
|
Value: value,
|
|
Exists: true,
|
|
Computed: computed,
|
|
}, nil
|
|
}
|
|
|
|
func (r *ConfigFieldReader) readPrimitive(
|
|
k string, schema *Schema) (FieldReadResult, error) {
|
|
raw, ok := r.Config.Get(k)
|
|
if !ok {
|
|
// Nothing in config, but we might still have a default from the schema
|
|
var err error
|
|
raw, err = schema.DefaultValue()
|
|
if err != nil {
|
|
return FieldReadResult{}, fmt.Errorf("%s, error loading default: %s", k, err)
|
|
}
|
|
|
|
if raw == nil {
|
|
return FieldReadResult{}, nil
|
|
}
|
|
}
|
|
|
|
var result string
|
|
if err := mapstructure.WeakDecode(raw, &result); err != nil {
|
|
return FieldReadResult{}, err
|
|
}
|
|
|
|
computed := r.Config.IsComputed(k)
|
|
returnVal, err := stringToPrimitive(result, computed, schema)
|
|
if err != nil {
|
|
return FieldReadResult{}, err
|
|
}
|
|
|
|
return FieldReadResult{
|
|
Value: returnVal,
|
|
Exists: true,
|
|
Computed: computed,
|
|
}, nil
|
|
}
|
|
|
|
func (r *ConfigFieldReader) readSet(
|
|
address []string, schema *Schema) (FieldReadResult, error) {
|
|
indexMap := make(map[string]int)
|
|
// Create the set that will be our result
|
|
set := schema.ZeroValue().(*Set)
|
|
|
|
raw, err := readListField(&nestedConfigFieldReader{r}, address, schema)
|
|
if err != nil {
|
|
return FieldReadResult{}, err
|
|
}
|
|
if !raw.Exists {
|
|
return FieldReadResult{Value: set}, nil
|
|
}
|
|
|
|
// If the list is computed, the set is necessarilly computed
|
|
if raw.Computed {
|
|
return FieldReadResult{
|
|
Value: set,
|
|
Exists: true,
|
|
Computed: raw.Computed,
|
|
}, nil
|
|
}
|
|
|
|
// Build up the set from the list elements
|
|
for i, v := range raw.Value.([]interface{}) {
|
|
// Check if any of the keys in this item are computed
|
|
computed := r.hasComputedSubKeys(
|
|
fmt.Sprintf("%s.%d", strings.Join(address, "."), i), schema)
|
|
|
|
code := set.add(v, computed)
|
|
indexMap[code] = i
|
|
}
|
|
|
|
r.indexMaps[strings.Join(address, ".")] = indexMap
|
|
|
|
return FieldReadResult{
|
|
Value: set,
|
|
Exists: true,
|
|
}, nil
|
|
}
|
|
|
|
// hasComputedSubKeys walks through a schema and returns whether or not the
|
|
// given key contains any subkeys that are computed.
|
|
func (r *ConfigFieldReader) hasComputedSubKeys(key string, schema *Schema) bool {
|
|
prefix := key + "."
|
|
|
|
switch t := schema.Elem.(type) {
|
|
case *Resource:
|
|
for k, schema := range t.Schema {
|
|
if r.Config.IsComputed(prefix + k) {
|
|
return true
|
|
}
|
|
|
|
if r.hasComputedSubKeys(prefix+k, schema) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// nestedConfigFieldReader is a funny little thing that just wraps a
|
|
// ConfigFieldReader to call readField when ReadField is called so that
|
|
// we don't recalculate the set rewrites in the address, which leads to
|
|
// an infinite loop.
|
|
type nestedConfigFieldReader struct {
|
|
Reader *ConfigFieldReader
|
|
}
|
|
|
|
func (r *nestedConfigFieldReader) ReadField(
|
|
address []string) (FieldReadResult, error) {
|
|
return r.Reader.readField(address, true)
|
|
}
|