Stop and remove container HANG in runc 1.2.x

[cospotato]

We have a scene need remove and delete container by containerd sdk. Code like below. It work in runc 1.1.x but hang in 1.2.x. I found a PR #3825. I built two binary with and without this PR. The without one work and with one hang.

Then i found another PR #4395 fixed some case introduce by #3825 . But i also tried the lates release v1.2.3. It's not work.

package main

import (
	"context"
	"log"
	"os"
	"syscall"
	"time"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/containers"
	"github.com/containerd/containerd/defaults"
	"github.com/containerd/containerd/namespaces"
	"github.com/containerd/containerd/runtime/restart"
	"github.com/containerd/errdefs"
	"github.com/pkg/errors"
	"google.golang.org/grpc"
	"google.golang.org/grpc/backoff"
	"google.golang.org/grpc/credentials/insecure"
)

func WithProcessTerminate(timeout time.Duration) containerd.ProcessDeleteOpts {
	return func(ctx context.Context, p containerd.Process) error {
		var cancel context.CancelFunc

		if timeout > 0 {
			ctx, cancel = context.WithTimeout(ctx, timeout)
		} else {
			ctx, cancel = context.WithCancel(ctx)
		}

		defer cancel()

		s, err := p.Wait(ctx)
		if err != nil {
			return err
		}

		if err := p.Kill(ctx, syscall.SIGTERM, containerd.WithKillAll); err != nil {
			// Kill might still return an IsNotFound error, even if it actually
			// killed the process.
			if errdefs.IsNotFound(err) {
				select {
				case <-ctx.Done():
					return ctx.Err()
				case <-s:
					return nil
				}
			}

			if errdefs.IsFailedPrecondition(err) {
				return nil
			}

			return err
		}
		// wait for the process to fully stop before letting the rest of the deletion complete
		<-s

		return nil
	}
}

func main() {
	if len(os.Args) < 2 {
		log.Fatal("need container id")
	}

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	// Cancel context is safe here because this context only controls the Dial
	// process, and once the connection is ready the context is useless.
	defer cancel()

	backoffConfig := backoff.DefaultConfig
	backoffConfig.MaxDelay = 3 * time.Second
	connParams := grpc.ConnectParams{
		Backoff: backoffConfig,
	}

	conn, err := grpc.DialContext(ctx, "unix:///run/containerd/containerd.sock",
		grpc.WithBlock(), // block until connection ready
		grpc.WithTransportCredentials(insecure.NewCredentials()),
		grpc.FailOnNonTempDialError(true),
		grpc.WithConnectParams(connParams),
		grpc.WithReturnConnectionError(),
		grpc.WithDefaultCallOptions(
			grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize),
			grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)),
	)

	if err != nil {
		log.Fatal(errors.Wrap(err, "dial containerd grpc failed"))
	}

	client, err := containerd.NewWithConn(conn)
	if err != nil {
		log.Fatal(errors.Wrap(err, "new containerd client failed"))
	}

	ctx1 := namespaces.WithNamespace(context.TODO(), "default")

	container, err := client.LoadContainer(ctx1, os.Args[1])
	if err != nil {
		if errdefs.IsNotFound(err) {
			log.Println("Already removed")
			return
		}

		log.Fatal(err)
	}

	log.Println("found container ", container.ID())

	_ = container.Update(ctx1, func(ctx context.Context, client *containerd.Client, c *containers.Container) error {
		delete(c.Labels, restart.StatusLabel)
		return nil
	})

	log.Println("removed restart label")

	task, err := container.Task(ctx1, nil)
	if err != nil && !errdefs.IsNotFound(err) {
		log.Fatal(err)
	}

	log.Printf("found container task %s, pid=%d", task.ID(), task.Pid())

	if err == nil {
		log.Println("start delete task")
		_, err = task.Delete(ctx1, WithProcessTerminate(0), containerd.WithProcessKill)
		if err != nil {
			log.Fatal(err)
		}
	}

	if err := container.Delete(ctx1); err != nil {
		log.Fatal(err)
	}
}