From a372a2af0b3bcf8d1045d679cbbf8166ef3bb352 Mon Sep 17 00:00:00 2001
From: tanyifeng <tanyifeng1@huawei.com>
Date: Thu, 21 Mar 2019 17:21:44 +0800
Subject: [PATCH 070/131] lxc: signal all process for shared container when
 container init exited

Signed-off-by: tanyifeng <tanyifeng1@huawei.com>
Signed-off-by: LiFeng <lifeng68@huawei.com>
---
 src/lxc/cgroups/cgfsng.c |   2 +-
 src/lxc/start.c          | 332 ++++++++++++++++++++-------------------
 2 files changed, 175 insertions(+), 159 deletions(-)

diff --git a/src/lxc/cgroups/cgfsng.c b/src/lxc/cgroups/cgfsng.c
index ab7ca35e..5ceb06b0 100644
--- a/src/lxc/cgroups/cgfsng.c
+++ b/src/lxc/cgroups/cgfsng.c
@@ -1056,7 +1056,7 @@ static int cgroup_rmdir(struct hierarchy **hierarchies,
 
 		ret = recursive_destroy(h->container_full_path);
 		if (ret < 0) {
-			SYSERROR("Failed to destroy \"%s\"", h->container_full_path);
+			SYSWARN("Failed to destroy \"%s\"", h->container_full_path);
 			return -1;
 		}
 
diff --git a/src/lxc/start.c b/src/lxc/start.c
index 1c9eb0ab..b14e46fd 100644
--- a/src/lxc/start.c
+++ b/src/lxc/start.c
@@ -941,6 +941,162 @@ out_close_maincmd_fd:
 	return -1;
 }
 
+void trim_line(char *s)
+{
+	size_t len;
+
+	len = strlen(s);
+	while ((len > 1) && (s[len - 1] == '\n'))
+		s[--len] = '\0';
+}
+
+static int _read_procs_file(const char *path, pid_t **pids, size_t *len)
+{
+	FILE *f;
+	char *line = NULL;
+	size_t sz = 0;
+
+	f = fopen_cloexec(path, "r");
+	if (!f)
+		return -1;
+
+	while (getline(&line, &sz, f) != -1) {
+		pid_t pid;
+		trim_line(line);
+		pid = (pid_t)atoll(line);
+		*pids = realloc(*pids, sizeof(pid_t) * (*len + 1));
+		(*pids)[*len] = pid;
+		(*len)++;
+	}
+
+	free(line);
+	fclose(f);
+	return 0;
+}
+
+static int _recursive_read_cgroup_procs(const char *dirpath, pid_t **pids, size_t *len)
+{
+	struct dirent *direntp;
+	DIR *dir;
+	int ret, failed = 0;
+	char pathname[PATH_MAX];
+
+	dir = opendir(dirpath);
+	if (!dir && errno != ENOENT) {
+		WARN("Failed to open \"%s\"", dirpath);
+		return 0;
+	}
+
+	while ((direntp = readdir(dir))) {
+		struct stat mystat;
+		int rc;
+
+		if (!strcmp(direntp->d_name, ".") ||
+		    !strcmp(direntp->d_name, ".."))
+			continue;
+
+		rc = snprintf(pathname, PATH_MAX, "%s/%s", dirpath, direntp->d_name);
+		if (rc < 0 || rc >= PATH_MAX) {
+			failed = 1;
+			continue;
+		}
+
+		if (strcmp(direntp->d_name, "cgroup.procs") == 0) {
+			if (_read_procs_file(pathname, pids, len)) {
+				failed = 1;
+
+			}
+			continue;
+		}
+
+		ret = lstat(pathname, &mystat);
+		if (ret) {
+			failed = 1;
+			continue;
+		}
+
+		if (S_ISDIR(mystat.st_mode)) {
+			if (_recursive_read_cgroup_procs(pathname, pids, len) < 0)
+				failed = 1;
+		}
+	}
+
+	ret = closedir(dir);
+	if (ret) {
+		WARN("Failed to close directory \"%s\"", dirpath);
+		failed = 1;
+	}
+
+	return failed ? -1 : 0;
+}
+
+int get_all_pids(struct cgroup_ops *cg_ops, pid_t **pids, size_t *len)
+{
+	char *devices_path = NULL;
+	int ret;
+
+	devices_path = must_make_path("/sys/fs/cgroup", "devices", cg_ops->container_cgroup, NULL);
+	if (!file_exists(devices_path)) {
+		free(devices_path);
+		return 0;
+	}
+
+	ret = _recursive_read_cgroup_procs(devices_path, pids, len);
+	free(devices_path);
+	return ret;
+}
+
+static int set_cgroup_freezer(struct cgroup_ops *cg_ops, const char *value)
+{
+	char *fullpath;
+	int ret;
+
+	fullpath = must_make_path("/sys/fs/cgroup", "freezer", cg_ops->container_cgroup, "freezer.state", NULL);
+	ret = lxc_write_to_file(fullpath, value, strlen(value), false, 0666);
+	free(fullpath);
+	return ret;
+}
+
+/* isulad: kill all process in container cgroup path */
+static void signal_all_processes(struct lxc_handler *handler)
+{
+	int ret;
+	struct cgroup_ops *cg_ops = handler->cgroup_ops;
+	pid_t *pids = NULL;
+	size_t len = 0, i;
+
+	ret = set_cgroup_freezer(cg_ops, "FROZEN");
+	if (ret < 0 && errno != ENOENT) {
+		WARN("cgroup_set frozen failed");
+	}
+
+	ret = get_all_pids(cg_ops, &pids, &len);
+	if (ret < 0) {
+		WARN("failed to get all pids");
+	}
+
+	for (i = 0; i < len; i++) {
+		ret = kill(pids[i], SIGKILL);
+		if (ret < 0 && errno != ESRCH) {
+			WARN("Can not kill process (pid=%d) with SIGKILL for container %s", pids[i], handler->name);
+		}
+	}
+
+	ret = set_cgroup_freezer(cg_ops, "THAWED");
+	if (ret < 0 && errno != ENOENT) {
+		WARN("cgroup_set thawed failed");
+	}
+
+	for (i = 0; i < len; i++) {
+		ret = lxc_wait_for_pid_status(pids[i]);
+		if (ret < 0 && errno != ECHILD) {
+			WARN("Failed to wait pid %d for container %s: %s", pids[i], handler->name, strerror(errno));
+		}
+	}
+
+	free(pids);
+}
+
 void lxc_fini(const char *name, struct lxc_handler *handler)
 {
 	int i, ret;
@@ -949,6 +1105,8 @@ void lxc_fini(const char *name, struct lxc_handler *handler)
 	char *namespaces[LXC_NS_MAX + 1];
 	size_t namespace_count = 0;
 	struct cgroup_ops *cgroup_ops = handler->cgroup_ops;
+	int retry_count = 0;
+	int max_retry = 10;
 
 	/* The STOPPING state is there for future cleanup code which can take
 	 * awhile.
@@ -1013,7 +1171,21 @@ void lxc_fini(const char *name, struct lxc_handler *handler)
 	while (namespace_count--)
 		free(namespaces[namespace_count]);
 
-	cgroup_ops->destroy(cgroup_ops, handler);
+	// if we shared pid namespace with others, should kill all processes within container cgroup
+	if (handler->conf->ns_share[LXC_NS_PID] != NULL) {
+		TRACE("Trying to kill all subprocess");
+		signal_all_processes(handler);
+		TRACE("Finished kill all subprocess");
+	}
+retry:
+	if (!cgroup_ops->destroy(cgroup_ops, handler)) {
+		if (retry_count < max_retry) {
+			usleep(100 * 1000); /* 100 millisecond */
+			retry_count++;
+			goto retry;
+		}
+		SYSERROR("Failed to destroy cgroup path for container: \"%s\"", handler->name);
+	}
 
 	if (handler->conf->reboot == REBOOT_NONE) {
 		/* For all new state clients simply close the command socket.
@@ -2494,162 +2666,6 @@ static void clean_resource_set_env(struct lxc_handler *handler)
 	/* End of environment variable setup for hooks. */
 }
 
-void trim_line(char *s)
-{
-	size_t len;
-
-	len = strlen(s);
-	while ((len > 1) && (s[len - 1] == '\n'))
-		s[--len] = '\0';
-}
-
-static int _read_procs_file(const char *path, pid_t **pids, size_t *len)
-{
-	FILE *f;
-	char *line = NULL;
-	size_t sz = 0;
-
-	f = fopen_cloexec(path, "r");
-	if (!f)
-		return -1;
-
-	while (getline(&line, &sz, f) != -1) {
-		pid_t pid;
-		trim_line(line);
-		pid = (pid_t)atoll(line);
-		*pids = realloc(*pids, sizeof(pid_t) * (*len + 1));
-		(*pids)[*len] = pid;
-		(*len)++;
-	}
-
-	free(line);
-	fclose(f);
-	return 0;
-}
-
-static int _recursive_read_cgroup_procs(const char *dirpath, pid_t **pids, size_t *len)
-{
-	struct dirent *direntp;
-	DIR *dir;
-	int ret, failed = 0;
-	char pathname[PATH_MAX];
-
-	dir = opendir(dirpath);
-	if (!dir && errno != ENOENT) {
-		WARN("Failed to open \"%s\"", dirpath);
-		return 0;
-	}
-
-	while ((direntp = readdir(dir))) {
-		struct stat mystat;
-		int rc;
-
-		if (!strcmp(direntp->d_name, ".") ||
-		    !strcmp(direntp->d_name, ".."))
-			continue;
-
-		rc = snprintf(pathname, PATH_MAX, "%s/%s", dirpath, direntp->d_name);
-		if (rc < 0 || rc >= PATH_MAX) {
-			failed = 1;
-			continue;
-		}
-
-		if (strcmp(direntp->d_name, "cgroup.procs") == 0) {
-			if (_read_procs_file(pathname, pids, len)) {
-				failed = 1;
-
-			}
-			continue;
-		}
-
-		ret = lstat(pathname, &mystat);
-		if (ret) {
-			failed = 1;
-			continue;
-		}
-
-		if (S_ISDIR(mystat.st_mode)) {
-			if (_recursive_read_cgroup_procs(pathname, pids, len) < 0)
-				failed = 1;
-		}
-	}
-
-	ret = closedir(dir);
-	if (ret) {
-		WARN("Failed to close directory \"%s\"", dirpath);
-		failed = 1;
-	}
-
-	return failed ? -1 : 0;
-}
-
-int get_all_pids(struct cgroup_ops *cg_ops, pid_t **pids, size_t *len)
-{
-	char *devices_path = NULL;
-	int ret;
-
-	devices_path = must_make_path("/sys/fs/cgroup", "devices", cg_ops->container_cgroup, NULL);
-	if (!file_exists(devices_path)) {
-		free(devices_path);
-		return 0;
-	}
-
-	ret = _recursive_read_cgroup_procs(devices_path, pids, len);
-	free(devices_path);
-	return ret;
-}
-
-static int set_cgroup_freezer(struct cgroup_ops *cg_ops, const char *value)
-{
-	char *fullpath;
-	int ret;
-
-	fullpath = must_make_path("/sys/fs/cgroup", "freezer", cg_ops->container_cgroup, "freezer.state", NULL);
-	ret = lxc_write_to_file(fullpath, value, strlen(value), false, 0666);
-	free(fullpath);
-	return ret;
-}
-
-/* isulad: kill all process in container cgroup path */
-static void signal_all_processes(struct lxc_handler *handler)
-{
-	int ret;
-	struct cgroup_ops *cg_ops = handler->cgroup_ops;
-	pid_t *pids = NULL;
-	size_t len = 0, i;
-
-	ret = set_cgroup_freezer(cg_ops, "FROZEN");
-	if (ret < 0 && errno != ENOENT) {
-		WARN("cgroup_set frozen failed");
-	}
-
-	ret = get_all_pids(cg_ops, &pids, &len);
-	if (ret < 0) {
-		WARN("failed to get all pids");
-	}
-
-	for (i = 0; i < len; i++) {
-		ret = kill(pids[i], SIGKILL);
-		if (ret < 0 && errno != ESRCH) {
-			WARN("Can not kill process (pid=%d) with SIGKILL for container %s", pids[i], handler->name);
-		}
-	}
-
-	ret = set_cgroup_freezer(cg_ops, "THAWED");
-	if (ret < 0 && errno != ENOENT) {
-		WARN("cgroup_set thawed failed");
-	}
-
-	for (i = 0; i < len; i++) {
-		ret = lxc_wait_for_pid_status(pids[i]);
-		if (ret < 0 && errno != ECHILD) {
-			WARN("Failed to wait pid %d for container %s: %s", pids[i], handler->name, strerror(errno));
-		}
-	}
-
-	free(pids);
-}
-
 /*isulad: do_lxcapi_clean_resource */
 int do_lxcapi_clean_resource(char *name, char *lxcpath, struct lxc_conf *conf, pid_t pid)
 {
@@ -2687,7 +2703,7 @@ retry:
 			retry_count++;
 			goto retry;
 		}
-		ERROR("Failed to destroy cgroup for container \"%s\".", handler->name);
+		SYSERROR("Failed to destroy cgroup path for container: \"%s\"", handler->name);
 		ret = -1;
 	}
 
-- 
2.23.0

