From 69d69a842a06671bc095bd4692dd8ea265cf6755 Mon Sep 17 00:00:00 2001
From: tanyifeng <tanyifeng1@huawei.com>
Date: Wed, 30 Jan 2019 17:44:19 +0800
Subject: [PATCH 060/131] using json-file to write console log of container

Signed-off-by: tanyifeng <tanyifeng1@huawei.com>
Signed-off-by: LiFeng <lifeng68@huawei.com>
---
 src/lxc/Makefile.am             |   2 +
 src/lxc/json/json_common.c      |  14 +-
 src/lxc/json/json_common.h      |   2 +
 src/lxc/json/logger_json_file.c | 243 ++++++++++++++++++++++++++++++++
 src/lxc/json/logger_json_file.h |  45 ++++++
 src/lxc/terminal.c              | 166 ++++++++++++++++++++--
 6 files changed, 457 insertions(+), 15 deletions(-)
 create mode 100644 src/lxc/json/logger_json_file.c
 create mode 100644 src/lxc/json/logger_json_file.h

diff --git a/src/lxc/Makefile.am b/src/lxc/Makefile.am
index 260a7eb0..698f8f95 100644
--- a/src/lxc/Makefile.am
+++ b/src/lxc/Makefile.am
@@ -47,6 +47,7 @@ noinst_HEADERS = attach.h \
 		 json/json_common.h \
 		 json/oci_runtime_hooks.h \
 		 json/oci_runtime_spec.h \
+		 json/logger_json_file.h \
 		 json/read-file.h \
 		 utils.h
 
@@ -149,6 +150,7 @@ liblxc_la_SOURCES = af_unix.c af_unix.h \
 		    json/json_common.c json/json_common.h \
 		    json/defs.h json/defs.c \
 		    json/oci_runtime_hooks.c json/oci_runtime_hooks.h \
+		    json/logger_json_file.c json/logger_json_file.h \
 		    json/oci_runtime_spec.c json/oci_runtime_spec.h \
 		    json/read-file.c json/read-file.h \
 		    $(LSM_SOURCES)
diff --git a/src/lxc/json/json_common.c b/src/lxc/json/json_common.c
index 8b91844d..e339ab3a 100755
--- a/src/lxc/json/json_common.c
+++ b/src/lxc/json/json_common.c
@@ -86,7 +86,7 @@ bool json_gen_init(yajl_gen *g, struct parser_context *ctx) {
 
     }
     yajl_gen_config(*g, yajl_gen_beautify, !(ctx->options & GEN_OPTIONS_SIMPLIFY));
-    yajl_gen_config(*g, yajl_gen_validate_utf8, 1);
+    yajl_gen_config(*g, yajl_gen_validate_utf8, !(ctx->options & GEN_OPTIONS_NOT_VALIDATE_UTF8));
     return true;
 }
 
@@ -1156,7 +1156,7 @@ json_map_string_string *make_json_map_string_string(yajl_val src, struct parser_
     return ret;
 }
 int append_json_map_string_string(json_map_string_string *map, const char *key, const char *val) {
-    size_t len;
+    size_t len, i;
     char **keys;
     char **vals;
 
@@ -1164,6 +1164,14 @@ int append_json_map_string_string(json_map_string_string *map, const char *key,
         return -1;
     }
 
+    for (i = 0; i < map->len; i++) {
+        if (strcmp(map->keys[i], key) == 0) {
+                free(map->values[i]);
+                map->values[i] = safe_strdup(val ? val : "");
+                return 0;
+        }
+    }
+
     if ((SIZE_MAX / sizeof(char *) - 1) < map->len) {
         return -1;
     }
@@ -1193,4 +1201,4 @@ int append_json_map_string_string(json_map_string_string *map, const char *key,
 
     map->len++;
     return 0;
-}
+}
\ No newline at end of file
diff --git a/src/lxc/json/json_common.h b/src/lxc/json/json_common.h
index 904fe3cd..eb8281c7 100755
--- a/src/lxc/json/json_common.h
+++ b/src/lxc/json/json_common.h
@@ -23,6 +23,8 @@ extern "C" {
 # define GEN_OPTIONS_ALLKEYVALUE 0x02
 //options to generate simplify(no indent) json string
 # define GEN_OPTIONS_SIMPLIFY 0x04
+//options not to validate utf8 data
+# define GEN_OPTIONS_NOT_VALIDATE_UTF8 0x08
 
 #define GEN_SET_ERROR_AND_RETURN(stat, err) { \
         if (!*(err)) {\
diff --git a/src/lxc/json/logger_json_file.c b/src/lxc/json/logger_json_file.c
new file mode 100644
index 00000000..4d781032
--- /dev/null
+++ b/src/lxc/json/logger_json_file.c
@@ -0,0 +1,243 @@
+// Generated from json-file.json. Do not edit!
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <string.h>
+#include <read-file.h>
+#include "securec.h"
+#include "logger_json_file.h"
+
+logger_json_file *make_logger_json_file(yajl_val tree, struct parser_context *ctx, parser_error *err) {
+    logger_json_file *ret = NULL;
+    *err = 0;
+    if (tree == NULL)
+        return ret;
+    ret = safe_malloc(sizeof(*ret));
+    {
+        yajl_val tmp = get_val(tree, "log", yajl_t_string);
+        if (tmp) {
+            char *str = YAJL_GET_STRING(tmp);
+            ret->log = (uint8_t *)safe_strdup(str ? str : "");
+            ret->log_len = str ? strlen(str) : 0;
+        }
+    }
+    {
+        yajl_val val = get_val(tree, "stream", yajl_t_string);
+        if (val) {
+            char *str = YAJL_GET_STRING(val);
+            ret->stream = safe_strdup(str ? str : "");
+        }
+    }
+    {
+        yajl_val val = get_val(tree, "time", yajl_t_string);
+        if (val) {
+            char *str = YAJL_GET_STRING(val);
+            ret->time = safe_strdup(str ? str : "");
+        }
+    }
+    {
+        yajl_val tmp = get_val(tree, "attrs", yajl_t_string);
+        if (tmp) {
+            char *str = YAJL_GET_STRING(tmp);
+            ret->attrs = (uint8_t *)safe_strdup(str ? str : "");
+            ret->attrs_len = str ? strlen(str) : 0;
+        }
+    }
+
+    if (tree->type == yajl_t_object && (ctx->options & PARSE_OPTIONS_STRICT)) {
+        int i;
+        for (i = 0; i < tree->u.object.len; i++)
+            if (strcmp(tree->u.object.keys[i], "log") &&
+                strcmp(tree->u.object.keys[i], "stream") &&
+                strcmp(tree->u.object.keys[i], "time") &&
+                strcmp(tree->u.object.keys[i], "attrs")) {
+                if (ctx->stderr > 0)
+                    fprintf(ctx->stderr, "WARNING: unknown key found: %s\n", tree->u.object.keys[i]);
+            }
+        }
+    return ret;
+}
+
+void free_logger_json_file(logger_json_file *ptr) {
+    if (!ptr)
+        return;
+    free(ptr->log);
+    ptr->log = NULL;
+    free(ptr->stream);
+    ptr->stream = NULL;
+    free(ptr->time);
+    ptr->time = NULL;
+    free(ptr->attrs);
+    ptr->attrs = NULL;
+    free(ptr);
+}
+
+yajl_gen_status gen_logger_json_file(yajl_gen g, logger_json_file *ptr, struct parser_context *ctx, parser_error *err) {
+    yajl_gen_status stat = yajl_gen_status_ok;
+    *err = 0;
+    stat = reformat_start_map(g);
+    if (yajl_gen_status_ok != stat)
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) || (ptr && ptr->log && ptr->log_len)) {
+        const char *str = "";
+        size_t len = 0;
+        stat = reformat_map_key(g, "log", strlen("log"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr && ptr->log) {
+            str = (const char *)ptr->log;
+            len = ptr->log_len;
+        }
+        stat = reformat_string(g, str, len);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr && ptr->stream)) {
+        char *str = "";
+        stat = reformat_map_key(g, "stream", strlen("stream"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr && ptr->stream) {
+            str = ptr->stream;
+        }
+        stat = reformat_string(g, str, strlen(str));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) ||(ptr && ptr->time)) {
+        char *str = "";
+        stat = reformat_map_key(g, "time", strlen("time"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr && ptr->time) {
+            str = ptr->time;
+        }
+        stat = reformat_string(g, str, strlen(str));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    if ((ctx->options & GEN_OPTIONS_ALLKEYVALUE) || (ptr && ptr->attrs && ptr->attrs_len)) {
+        const char *str = "";
+        size_t len = 0;
+        stat = reformat_map_key(g, "attrs", strlen("attrs"));
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+        if (ptr && ptr->attrs) {
+            str = (const char *)ptr->attrs;
+            len = ptr->attrs_len;
+        }
+        stat = reformat_string(g, str, len);
+        if (yajl_gen_status_ok != stat)
+            GEN_SET_ERROR_AND_RETURN(stat, err);
+    }
+    stat = reformat_end_map(g);
+    if (yajl_gen_status_ok != stat)
+        GEN_SET_ERROR_AND_RETURN(stat, err);
+    return yajl_gen_status_ok;
+}
+
+
+logger_json_file *logger_json_file_parse_file(const char *filename, struct parser_context *ctx, parser_error *err) {
+    logger_json_file *ptr;
+    size_t filesize;
+    char *content;
+
+    if (!filename || !err)
+        return NULL;
+
+    *err = NULL;
+    content = read_file(filename, &filesize);
+    if (content == NULL) {
+        if (asprintf(err, "cannot read the file: %s", filename) < 0)
+            *err = safe_strdup("error allocating memory");
+        return NULL;
+    }
+    ptr = logger_json_file_parse_data(content, ctx, err);
+    free(content);
+    return ptr;
+}
+
+logger_json_file *logger_json_file_parse_file_stream(FILE *stream, struct parser_context *ctx, parser_error *err) {
+    logger_json_file *ptr;
+    size_t filesize;
+    char *content ;
+
+    if (!stream || !err)
+        return NULL;
+
+    *err = NULL;
+    content = fread_file(stream, &filesize);
+    if (content == NULL) {
+        *err = safe_strdup("cannot read the file");
+        return NULL;
+    }
+    ptr = logger_json_file_parse_data(content, ctx, err);
+    free(content);
+    return ptr;
+}
+
+logger_json_file *logger_json_file_parse_data(const char *jsondata, struct parser_context *ctx, parser_error *err) {
+    logger_json_file *ptr;
+    yajl_val tree;
+    char errbuf[1024];
+    struct parser_context tmp_ctx;
+
+    if (!jsondata || !err)
+        return NULL;
+
+    *err = NULL;
+    if (!ctx) {
+       ctx = &tmp_ctx;
+       memset(&tmp_ctx, 0, sizeof(tmp_ctx));
+    }
+    tree = yajl_tree_parse(jsondata, errbuf, sizeof(errbuf));
+    if (tree == NULL) {
+        if (asprintf(err, "cannot parse the data: %s", errbuf) < 0)
+            *err = safe_strdup("error allocating memory");
+        return NULL;
+    }
+    ptr = make_logger_json_file(tree, ctx, err);
+    yajl_tree_free(tree);
+    return ptr;
+}
+char *logger_json_file_generate_json(logger_json_file *ptr, struct parser_context *ctx, parser_error *err) {
+    yajl_gen g = NULL;
+    struct parser_context tmp_ctx;
+    const unsigned char *gen_buf = NULL;
+    char *json_buf = NULL;
+    size_t gen_len = 0;
+
+    if (!ptr || !err)
+        return NULL;
+
+    *err = NULL;
+    if (!ctx) {
+        ctx = &tmp_ctx;
+        memset(&tmp_ctx, 0, sizeof(tmp_ctx));
+    }
+
+    if (!json_gen_init(&g, ctx)) {
+        *err = safe_strdup("Json_gen init failed");
+        goto out;
+    }
+    if (yajl_gen_status_ok != gen_logger_json_file(g, ptr, ctx, err)) {
+        if (!*err)
+            *err = safe_strdup("Failed to generate json");
+        goto free_out;
+    }
+    yajl_gen_get_buf(g, &gen_buf, &gen_len);
+    if (!gen_buf) {
+        *err = safe_strdup("Error to get generated json");
+        goto free_out;
+    }
+
+    json_buf = safe_malloc(gen_len + 1);
+    memcpy(json_buf, gen_buf, gen_len);
+    json_buf[gen_len] = '\0';
+
+free_out:
+    yajl_gen_clear(g);
+    yajl_gen_free(g);
+out:
+    return json_buf;
+}
diff --git a/src/lxc/json/logger_json_file.h b/src/lxc/json/logger_json_file.h
new file mode 100644
index 00000000..ad5af7b4
--- /dev/null
+++ b/src/lxc/json/logger_json_file.h
@@ -0,0 +1,45 @@
+// Generated from json-file.json. Do not edit!
+#ifndef LOGGER_JSON_FILE_SCHEMA_H
+#define LOGGER_JSON_FILE_SCHEMA_H
+
+#include <sys/types.h>
+#include <stdint.h>
+#include "json_common.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+    uint8_t *log;
+    size_t log_len;
+
+    char *stream;
+
+    char *time;
+
+    uint8_t *attrs;
+    size_t attrs_len;
+
+}
+logger_json_file;
+
+void free_logger_json_file(logger_json_file *ptr);
+
+logger_json_file *make_logger_json_file(yajl_val tree, struct parser_context *ctx, parser_error *err);
+
+yajl_gen_status gen_logger_json_file(yajl_gen g, logger_json_file *ptr, struct parser_context *ctx, parser_error *err);
+
+logger_json_file *logger_json_file_parse_file(const char *filename, struct parser_context *ctx, parser_error *err);
+
+logger_json_file *logger_json_file_parse_file_stream(FILE *stream, struct parser_context *ctx, parser_error *err);
+
+logger_json_file *logger_json_file_parse_data(const char *jsondata, struct parser_context *ctx, parser_error *err);
+
+char *logger_json_file_generate_json(logger_json_file *ptr, struct parser_context *ctx, parser_error *err);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/lxc/terminal.c b/src/lxc/terminal.c
index 252a644f..602d43d7 100644
--- a/src/lxc/terminal.c
+++ b/src/lxc/terminal.c
@@ -47,6 +47,7 @@
 #include "start.h"
 #include "syscall_wrappers.h"
 #include "terminal.h"
+#include "logger_json_file.h"
 #include "utils.h"
 
 #if HAVE_PTY_H
@@ -298,7 +299,7 @@ static int lxc_terminal_rotate_log_file(struct lxc_terminal *terminal)
 	return lxc_terminal_create_log_file(terminal);
 }
 
-static int lxc_terminal_write_log_file(struct lxc_terminal *terminal, char *buf,
+static int lxc_terminal_rotate_write_data(struct lxc_terminal *terminal, const char *buf,
 				       int bytes_read)
 {
 	int ret;
@@ -357,16 +358,6 @@ static int lxc_terminal_write_log_file(struct lxc_terminal *terminal, char *buf,
 	if (bytes_read <= space_left)
 		return lxc_write_nointr(terminal->log_fd, buf, bytes_read);
 
-	/* There's not enough space left but at least write as much as we can
-	 * into the old log file.
-	 */
-	ret = lxc_write_nointr(terminal->log_fd, buf, space_left);
-	if (ret < 0)
-		return -1;
-
-	/* Calculate how many bytes we still need to write. */
-	bytes_read -= space_left;
-
 	/* There'd be more to write but we aren't instructed to rotate the log
 	 * file so simply return. There's no error on our side here.
 	 */
@@ -404,6 +395,151 @@ static int lxc_terminal_write_log_file(struct lxc_terminal *terminal, char *buf,
 	return bytes_read;
 }
 
+/* get time buffer */
+static bool get_time_buffer(struct timespec *timestamp, char *timebuffer,
+                     size_t maxsize)
+{
+    struct tm tm_utc = { 0 };
+    int32_t nanos = 0;
+    time_t seconds;
+
+    if (!timebuffer || !maxsize) {
+        return false;
+    }
+
+    seconds = (time_t)timestamp->tv_sec;
+    gmtime_r(&seconds, &tm_utc);
+    strftime(timebuffer, maxsize, "%Y-%m-%dT%H:%M:%S", &tm_utc);
+
+    nanos = (int32_t)timestamp->tv_nsec;
+    sprintf(timebuffer + strlen(timebuffer), ".%09dZ", nanos);
+
+    return true;
+}
+
+/* get now time buffer */
+static bool get_now_time_buffer(char *timebuffer, size_t maxsize)
+{
+    int err = 0;
+    struct timespec ts;
+
+    err = clock_gettime(CLOCK_REALTIME, &ts);
+    if (err != 0) {
+        ERROR("failed to get time");
+        return false;
+    }
+
+    return get_time_buffer(&ts, timebuffer, maxsize);
+}
+
+static ssize_t lxc_logger_write(struct lxc_terminal *terminal, const char *buf,
+				       int bytes_read)
+{
+	logger_json_file *msg = NULL;
+	ssize_t ret = -1;
+	size_t len;
+	char *json = NULL, timebuffer[64];
+	parser_error err = NULL;
+	struct parser_context ctx = { GEN_OPTIONS_SIMPLIFY | GEN_OPTIONS_NOT_VALIDATE_UTF8, stderr };
+
+	msg = calloc(sizeof(logger_json_file), 1);
+	if (!msg) {
+		return -errno;
+	}
+	msg->log = calloc(bytes_read, 1);
+	if (!msg->log) {
+		goto cleanup;
+	}
+	memcpy(msg->log, buf, bytes_read);
+	msg->log_len = bytes_read;
+	msg->stream = strdup("stdout");
+
+	get_now_time_buffer(timebuffer, sizeof(timebuffer));
+	msg->time = strdup(timebuffer);
+
+	json = logger_json_file_generate_json(msg, &ctx, &err);
+	if (!json) {
+		ERROR("Failed to generate json: %s", err);
+		goto cleanup;
+	}
+	len = strlen(json);
+	json[len] = '\n';
+	ret = lxc_terminal_rotate_write_data(terminal, json, len + 1);
+cleanup:
+	free(json);
+	free_logger_json_file(msg);
+	free(err);
+	return ret;
+}
+
+static int lxc_terminal_write_log_file(struct lxc_terminal *terminal, char *buf,
+				       int bytes_read)
+{
+#define __BUF_CACHE_SIZE (16 * LXC_TERMINAL_BUFFER_SIZE)
+	static char cache[__BUF_CACHE_SIZE];
+	static int size = 0;
+	int upto, index;
+	int begin = 0, buf_readed = 0,  buf_left = 0;
+	int ret;
+
+	if (buf != NULL && bytes_read > 0) {
+		/* Work out how much more data we are okay with reading this time. */
+		upto = size + bytes_read;
+		if (upto > __BUF_CACHE_SIZE) {
+			upto = __BUF_CACHE_SIZE;
+		}
+
+		if (upto > size) {
+			buf_readed = upto - size;
+			memcpy(cache + size, buf, buf_readed);
+			buf_left = bytes_read - buf_readed;
+			size += buf_readed;
+		}
+	}
+
+	// If we have no data to log, and there's no more coming, we're done.
+	if (size == 0)
+		return 0;
+
+	// Break up the data that we've buffered up into lines, and log each in turn.
+	for (index = 0; index < size; index++) {
+		if (cache[index] == '\n') {
+			ret = lxc_logger_write(terminal, cache + begin, index - begin + 1);
+			if (ret < 0) {
+				WARN("Failed to log msg");
+			}
+			begin = index + 1;
+		}
+	}
+	/* If there's no more coming, or the buffer is full but
+	 * has no newlines, log whatever we haven't logged yet,
+	 * noting that it's a partial log line. */
+	if (buf == NULL || (begin == 0 && size == __BUF_CACHE_SIZE)) {
+		if (begin < size) {
+			ret = lxc_logger_write(terminal, cache + begin, index - begin + 1);
+			if (ret < 0) {
+				WARN("Failed to log msg");
+			}
+			begin = 0;
+			size = 0;
+		}
+		if (buf == NULL) {
+			return 0;
+		}
+	}
+	/* Move any unlogged data to the front of the buffer in preparation for another read. */
+	if (begin > 0) {
+		memcpy(cache, cache + begin, size - begin);
+		size -= begin;
+	}
+	/* Move left data to cache buffer */
+	if (buf_left > 0) {
+		memcpy(cache + size, buf + buf_readed, buf_left);
+		size += buf_left;
+	}
+	return 0;
+}
+
 /* isulad: forward data to all fifos */
 static void lxc_forward_data_to_fifo(struct lxc_list *list, char *buf, int r)
 {
@@ -437,7 +573,7 @@ int lxc_terminal_io_cb(int fd, uint32_t events, void *data,
 		       struct lxc_epoll_descr *descr)
 {
 	struct lxc_terminal *terminal = data;
-	char buf[LXC_TERMINAL_BUFFER_SIZE];
+	char buf[2 * LXC_TERMINAL_BUFFER_SIZE];
 	int r, w, w_log, w_rbuf;
 
 	w = r = lxc_read_nointr(fd, buf, sizeof(buf));
@@ -447,6 +583,12 @@ int lxc_terminal_io_cb(int fd, uint32_t events, void *data,
 
 		if (fd == terminal->master) {
 			terminal->master = -EBADF;
+			/* write remained buffer to terminal log */
+			if (terminal->log_fd >= 0) {
+				w_log = lxc_terminal_write_log_file(terminal, NULL, 0);
+				if (w_log < 0)
+					TRACE("Failed to write %d bytes to terminal log", r);
+			}
 		} else if (fd == terminal->peer) {
 			if (terminal->tty_state) {
 				lxc_terminal_signal_fini(terminal->tty_state);
-- 
2.23.0

