busy-soccer-65968
04/21/2021, 10:45 PMbored-oyster-3147
04/21/2021, 11:00 PMUp
commands concurrently within the same process. I'm not sure if Destroy
would also be caught by that since it runs against your state and doesn't do much in-process. Hopefully someone else can answerlemon-agent-27707
04/21/2021, 11:06 PMbusy-soccer-65968
04/21/2021, 11:07 PMfunc removeOldPulumiStacks(projectName string) {
ctx := context.Background()
ws, err := auto.NewLocalWorkspace(ctx, auto.Project(workspace.Project{
Name: tokens.PackageName(projectName),
Runtime: workspace.NewProjectRuntimeInfo("go", nil),
}))
if err != nil {
fmt.Println("Failed creating workspace from project name:", projectName)
fmt.Println(err.Error())
return
}
stacks, err := ws.ListStacks(ctx)
if err != nil {
fmt.Println("Failed listing stacks for project:", projectName)
fmt.Println(err.Error())
return
}
sevenDaysAgo := time.Now().Add(-7 * time.Hour * 24).UTC()
ignoreStacks := []string{"prod", "build", "stage", "production", "staging"}
var wg sync.WaitGroup
OUTER:
for _, stack := range stacks {
// Do nothing for stacks in our ignoreStacks slice
// This is just an extra precaution
for _, s := range ignoreStacks {
if strings.Contains(stack.Name, s) {
continue OUTER
}
}
if stack.UpdateInProgress {
continue
}
lastUpdated, err := time.Parse(time.RFC3339, stack.LastUpdate)
if err != nil {
fmt.Printf("Error parsing last update time for stack:\n%+v\n", stack)
fmt.Println(err.Error())
return
}
if lastUpdated.Before(sevenDaysAgo) && (strings.Contains(stack.Name, "ephemeral") || strings.Contains(stack.Name, "test")) {
wg.Add(1)
go deleteStack(ctx, stack, ws, &wg)
}
}
wg.Wait()
}
func deleteStack(ctx context.Context, stack auto.StackSummary, ws auto.Workspace, wg *sync.WaitGroup) {
defer wg.Done()
fmt.Println("Deleting stack:", stack.Name)
s, err := auto.SelectStack(ctx, stack.Name, ws)
if err != nil {
fmt.Printf("Error selecting stack:\n%+v\n", stack)
fmt.Println(err.Error())
return
}
_, err = s.Destroy(ctx)
if err != nil {
fmt.Printf("Error destroying stack:\n%+v\n", stack)
fmt.Println(err.Error())
return
}
err = ws.RemoveStack(ctx, stack.Name)
if err != nil {
fmt.Printf("Failed removing stack:\n%+v\n", stack)
fmt.Println(err.Error())
return
}
fmt.Println("Removed stack:", stack.Name)
}
go
then all runs as expectedbored-oyster-3147
04/21/2021, 11:23 PMpulumi stack select
. The master
branch still has the workspace selecting the stack before taking an action instead of using the --stack
argument, which I know was part of a recent changesetbusy-soccer-65968
04/21/2021, 11:26 PMbored-oyster-3147
04/21/2021, 11:35 PM--stack
change but now I can't find it, and it doesn't look like it's in 3.0 yet eitherbusy-soccer-65968
04/21/2021, 11:41 PMlemon-agent-27707
04/22/2021, 12:07 AMfor _, s := range stacknames {
stack := auto.SelectStack()...
go stack.Destroy()...
}
red-match-15116
04/22/2021, 1:06 AMbusy-soccer-65968
04/22/2021, 2:37 PM▶ pulumi version
v3.0.0
bored-oyster-3147
04/22/2021, 3:15 PM--stack
directly on the destroy
command than it is still affecting global state of the workspace and is thus a race conditionbusy-soccer-65968
04/22/2021, 3:15 PMbored-oyster-3147
04/22/2021, 3:17 PMbusy-soccer-65968
04/22/2021, 3:17 PMred-match-15116
04/22/2021, 3:18 PMbored-oyster-3147
04/22/2021, 3:20 PMred-match-15116
04/22/2021, 3:20 PMbusy-soccer-65968
04/22/2021, 3:20 PMred-match-15116
04/22/2021, 8:10 PMbusy-soccer-65968
04/22/2021, 8:41 PMred-match-15116
04/22/2021, 8:52 PMbusy-soccer-65968
04/22/2021, 8:57 PMstderr: error: [409] Conflict: Another update is currently in progress.
updating go SDK nowpending_operations
when I ctrl-c. but that seems like it.red-match-15116
04/22/2021, 9:01 PMbusy-soccer-65968
04/22/2021, 9:10 PMpulumi destroy
or pulumi stack rm
I believe it blocks until completionpulumi destroy -s my stack
seems to believe most of them have pending_operations
this could be from me mistakenly running 3.1
CLI with 3.0
sdk. I will test with spinning up quite a few fresh test stacksred-match-15116
04/22/2021, 9:20 PMbusy-soccer-65968
04/22/2021, 9:30 PMpending_operations
from messing up the first try. I made 5 test stacks all with resources and then tested my code against them and all 5 deleted in parallel and also blocked until Stackselect --> destroy ---> remove completed per stack. and I verified inside web GUI.
My only real suggestion would be maybe investigate failed destroys for `pending_operations`` otherwise I think i'll write off the problem as my own mistake.
Thanks again for all the help. I'll be sure to let you all know if I find anything else fishy.red-match-15116
04/22/2021, 9:33 PMstack.Export()
, remove the pending operations key and then do stack.Import()
busy-soccer-65968
04/22/2021, 9:34 PMFROM scratch
docker container to run this inred-match-15116
04/22/2021, 9:41 PMyah, tbh it’s ~ 50 stacks. and copy and pasting stack names them all and writing new code would take longer than manually doing it for me. also, all the sub resources are already deleted from a separate process so I can just force remove the stack without worrying about dangling resourcesJust spitballing here, you could list stacks: https://pkg.go.dev/github.com/pulumi/pulumi/sdk/v3/go/auto#LocalWorkspace.ListStacks And then loop through and export: https://pkg.go.dev/github.com/pulumi/pulumi/sdk/v3/go/auto#LocalWorkspace.ExportStack remove the pending ops and then import: https://pkg.go.dev/github.com/pulumi/pulumi/sdk/v3/go/auto#LocalWorkspace.ImportStack
busy-soccer-65968
04/22/2021, 9:42 PM