DAOS-17321 ddb: Checksum management with ddb

First commit.

Signed-off-by: Cedric Koch-Hofer <[email protected]>

Author: kanard38

site_scons/prereq_tools/base.py

@@ -518,6 +518,7 @@ def __init__(self, env, opts):
         opts.Add(EnumVariable('WARNING_LEVEL', "Set default warning level", 'error',
                               ['warning', 'warn', 'error'], ignorecase=2))
         opts.Add(('SANITIZERS', 'Instrument C code with google sanitizers', None))
+        opts.Add(BoolVariable('CMOCKA_FILTER_SUPPORTED', 'Allows to filter cmocka tests', False))
 
         opts.Update(self.__env)
 

site_scons/site_tools/compiler_setup.py

@@ -54,7 +54,7 @@ def _base_setup(env):
 
     # Turn on -Wall first, then DESIRED_FLAGS may disable some of the options
     # that this brings in.
-    env.Append(CCFLAGS=['-g', '-Wextra', '-Wshadow', '-Wall', '-fpic'])
+    env.Append(CCFLAGS=['-g3', '-Wextra', '-Wshadow', '-Wall', '-fpic'])
 
     env.AppendIfSupported(CCFLAGS=DESIRED_FLAGS)
 
@@ -106,7 +106,10 @@ def _base_setup(env):
         env.AppendUnique(CPPDEFINES={'FAULT_INJECTION': '1'})
         env.AppendUnique(CPPDEFINES={'BUILD_PIPELINE': '1'})
 
-    env.AppendUnique(CPPDEFINES={'CMOCKA_FILTER_SUPPORTED': '0'})
+    if env['CMOCKA_FILTER_SUPPORTED']:
+        env.AppendUnique(CPPDEFINES={'CMOCKA_FILTER_SUPPORTED': '1'})
+    else:
+        env.AppendUnique(CPPDEFINES={'CMOCKA_FILTER_SUPPORTED': '0'})
 
     env.AppendUnique(CPPDEFINES='_GNU_SOURCE')
 

src/control/cmd/ddb/commands_wrapper.go

@@ -349,3 +349,15 @@ func ddbDtxAggr(ctx *DdbContext, path string, cmt_time uint64, cmt_date string)
 	/* Run the c code command */
 	return daosError(C.ddb_run_dtx_aggr(&ctx.ctx, &options))
 }
+
+func ddbCsumDump(ctx *DdbContext, path string, dst string, epoch uint64) error {
+	/* Set up the options */
+	options := C.struct_csum_dump_options{}
+	options.path = C.CString(path)
+	defer freeString(options.path)
+	options.dst = C.CString(dst)
+	defer freeString(options.dst)
+	options.epoch = C.uint64_t(epoch)
+	/* Run the c code command */
+	return daosError(C.ddb_run_csum_dump(&ctx.ctx, &options))
+}

src/control/cmd/ddb/ddb_commands.go

@@ -442,4 +442,28 @@ the path must include the extent, otherwise, it must not.`,
 		},
 		Completer: nil,
 	})
+	// Command csum_dump
+	app.AddCommand(&grumble.Command{
+		Name:    "csum_dump",
+		Aliases: nil,
+		Help:    "Dump visible checksum(s)",
+		LongHelp: `Dump visible checksum(s) to the screen or in a file.  The vos
+path should be a complete path, including the akey and if the value is an array
+value it should include the extent. If a path to a file was provided then the
+value(s) will be written to the file, else it will be printed to the screen.
+With array values it is possible to define the maximal epoch of the visible record
+extent to select`,
+		HelpGroup: "vos",
+		Args: func(a *grumble.Args) {
+			a.String("path", "VOS tree path to dump.")
+			a.String("dst", "Optional, destination vos tree path to a value.", grumble.Default(""))
+		},
+		Flags: func(f *grumble.Flags) {
+			f.Uint64("e", "epoch", math.MaxUint64, "Maximal epoch of the visible array value to select (default EPOCH_MAX).")
+		},
+		Run: func(c *grumble.Context) error {
+			return ddbCsumDump(ctx, c.Args.String("path"), c.Args.String("dst"), c.Flags.Uint64("epoch"))
+		},
+		Completer: nil,
+	})
 }

src/include/daos_srv/vos.h

@@ -723,6 +723,16 @@ vos_obj_fetch_ex(daos_handle_t coh, daos_unit_oid_t oid, daos_epoch_t epoch,
 		 uint64_t flags, daos_key_t *dkey, unsigned int iod_nr,
 		 daos_iod_t *iods, d_sg_list_t *sgls, struct dtx_handle *dth);
 
+// FIXME DAOS-17321 -- Old checksum fetch API, to be removed later
+#if 0
+/**
+ * TODO DAOS-17321
+ */
+int
+vos_csum_fetch(daos_handle_t coh, daos_key_t *dkey, daos_unit_oid_t oid, daos_iod_t *iod,
+	       struct daos_recx_ep_list *rel, struct dcs_ci_list *cil);
+#endif
+
 /**
  * Update records for the specified object.
  * If input buffer is not provided in \a sgl, then this function returns
@@ -905,6 +915,7 @@ vos_obj_mark_corruption(daos_handle_t coh, daos_epoch_t epoch, uint32_t pm_ver,
  * I/O and release resources.
  *
  * TODO: add more detail descriptions for punched or missing records.
+ * TODO DAOS-17321 -- Add description for csum handling.
  *
  * \param[in] coh	Container open handle
  * \param[in] oid	Object ID

src/include/daos_srv/vos_types.h

@@ -258,49 +258,51 @@ typedef enum {
 
 enum {
 	/** Conditional Op: Punch key if it exists, fail otherwise */
-	VOS_OF_COND_PUNCH		= DAOS_COND_PUNCH,
+	VOS_OF_COND_PUNCH = DAOS_COND_PUNCH,
 	/** Conditional Op: Insert dkey if it doesn't exist, fail otherwise */
-	VOS_OF_COND_DKEY_INSERT		= DAOS_COND_DKEY_INSERT,
+	VOS_OF_COND_DKEY_INSERT = DAOS_COND_DKEY_INSERT,
 	/** Conditional Op: Update dkey if it exists, fail otherwise */
-	VOS_OF_COND_DKEY_UPDATE		= DAOS_COND_DKEY_UPDATE,
+	VOS_OF_COND_DKEY_UPDATE = DAOS_COND_DKEY_UPDATE,
 	/** Conditional Op: Fetch dkey if it exists, fail otherwise */
-	VOS_OF_COND_DKEY_FETCH		= DAOS_COND_DKEY_FETCH,
+	VOS_OF_COND_DKEY_FETCH = DAOS_COND_DKEY_FETCH,
 	/** Conditional Op: Insert akey if it doesn't exist, fail otherwise */
-	VOS_OF_COND_AKEY_INSERT		= DAOS_COND_AKEY_INSERT,
+	VOS_OF_COND_AKEY_INSERT = DAOS_COND_AKEY_INSERT,
 	/** Conditional Op: Update akey if it exists, fail otherwise */
-	VOS_OF_COND_AKEY_UPDATE		= DAOS_COND_AKEY_UPDATE,
+	VOS_OF_COND_AKEY_UPDATE = DAOS_COND_AKEY_UPDATE,
 	/** Conditional Op: Fetch akey if it exists, fail otherwise */
-	VOS_OF_COND_AKEY_FETCH		= DAOS_COND_AKEY_FETCH,
+	VOS_OF_COND_AKEY_FETCH = DAOS_COND_AKEY_FETCH,
 	/** Indicates akey conditions are specified in iod_flags */
-	VOS_OF_COND_PER_AKEY		= DAOS_COND_PER_AKEY,
+	VOS_OF_COND_PER_AKEY = DAOS_COND_PER_AKEY,
 	/* critical update - skip checks on SCM system/held space */
-	VOS_OF_CRIT			= (1 << 8),
+	VOS_OF_CRIT = (1 << 8),
 	/** Instead of update or punch of extents, remove all extents
 	 * under the specified range. Intended for internal use only.
 	 */
-	VOS_OF_REMOVE			= (1 << 9),
+	VOS_OF_REMOVE = (1 << 9),
 	/* only query iod_size */
-	VOS_OF_FETCH_SIZE_ONLY		= (1 << 10),
+	VOS_OF_FETCH_SIZE_ONLY = (1 << 10),
 	/* query recx list */
-	VOS_OF_FETCH_RECX_LIST		= (1 << 11),
+	VOS_OF_FETCH_RECX_LIST = (1 << 11),
 	/* only set read TS */
-	VOS_OF_FETCH_SET_TS_ONLY	= (1 << 12),
+	VOS_OF_FETCH_SET_TS_ONLY = (1 << 12),
 	/* check the target (obj/dkey/akey) existence */
-	VOS_OF_FETCH_CHECK_EXISTENCE	= (1 << 13),
+	VOS_OF_FETCH_CHECK_EXISTENCE = (1 << 13),
 	/** Set when propagating a punch that results in empty subtree */
-	VOS_OF_PUNCH_PROPAGATE		= (1 << 14),
+	VOS_OF_PUNCH_PROPAGATE = (1 << 14),
 	/** replay punch (underwrite) */
-	VOS_OF_REPLAY_PC		= (1 << 15),
+	VOS_OF_REPLAY_PC = (1 << 15),
 	/** Dedup update mode */
-	VOS_OF_DEDUP			= (1 << 16),
+	VOS_OF_DEDUP = (1 << 16),
 	/** Dedup update with memcmp verify mode */
-	VOS_OF_DEDUP_VERIFY		= (1 << 17),
+	VOS_OF_DEDUP_VERIFY = (1 << 17),
 	/** Ignore fetch only used by shadow fetch to ignore the evt fetch */
-	VOS_OF_SKIP_FETCH		= (1 << 18),
+	VOS_OF_SKIP_FETCH = (1 << 18),
 	/** Operation on EC object (currently only applies to update) */
-	VOS_OF_EC			= (1 << 19),
+	VOS_OF_EC = (1 << 19),
 	/** Update from rebuild */
-	VOS_OF_REBUILD			= (1 << 20),
+	VOS_OF_REBUILD = (1 << 20),
+	/* only query iod_size */
+	VOS_OF_FETCH_CSUM = (1 << 21),
 };
 
 enum {

src/utils/ddb/SConscript

@@ -55,6 +55,10 @@ def scons():
     ddblib = denv.d_library('ddb', src, LIBS=libs)
     denv.Install('$PREFIX/lib64/daos_srv/', ddblib)
 
+    # FIXME DAOS-17321 -- Conditional build for ddb legacies
+    ddbexe = denv.d_program('ddb_leg', src + ['ddb_entry.c'], LIBS=libs)
+    denv.Install('$PREFIX/bin', ddbexe)
+
     # tests
     SConscript('tests/SConscript', exports=['denv'])
 

src/utils/ddb/ddb.c

@@ -49,6 +49,7 @@
 #define COMMAND_NAME_DTX_STAT                "dtx_stat"
 #define COMMAND_NAME_PROV_MEM                "prov_mem"
 #define COMMAND_NAME_DTX_AGGR                "dtx_aggr"
+#define COMMAND_NAME_CSUM_DUMP               "csum_dump"
 
 /* Parse command line options for the 'ls' command */
 static int
@@ -996,6 +997,56 @@ dtx_aggr_option_parse(struct ddb_ctx *ctx, struct dtx_aggr_options *cmd_args, ui
 	return 0;
 }
 
+/* Parse command line options for the 'dtx_aggr' command */
+static int
+csum_dump_option_parse(struct ddb_ctx *ctx, struct csum_dump_options *cmd_args, uint32_t argc,
+		       char **argv)
+{
+	int           opt;
+	char         *options_short = "e:";
+	int           index         = 0;
+	char         *endptr;
+	struct option options_long[] = {{"epoch", required_argument, NULL, 'e'}, {NULL}};
+
+	memset(cmd_args, 0, sizeof(*cmd_args));
+
+	/* Restart getopt */
+	optind          = 1;
+	opterr          = 0;
+	cmd_args->epoch = DAOS_EPOCH_MAX;
+	while ((opt = getopt_long(argc, argv, options_short, options_long, &index)) != -1) {
+		switch (opt) {
+		case 'e':
+			errno           = 0;
+			cmd_args->epoch = strtoull(optarg, &endptr, 10);
+			if (errno != 0 || endptr == optarg || *endptr != '\0') {
+				ddb_error(ctx, "'--epoch' option arg format is invalid\n");
+				return -DER_INVAL;
+			}
+			break;
+		case '?':
+			ddb_printf(ctx, "Unknown option: '%c'\n", optopt);
+			break;
+		default:
+			return -DER_INVAL;
+		}
+	}
+
+	index          = optind;
+	cmd_args->path = NULL;
+	if (argc - index > 0 && *argv[index] != '\0') {
+		cmd_args->path = argv[index];
+		index++;
+	}
+
+	if (argc - index > 0) {
+		ddb_errorf(ctx, "Unexpected argument: %s\n", argv[index]);
+		return -DER_INVAL;
+	}
+
+	return 0;
+}
+
 int
 ddb_parse_cmd_args(struct ddb_ctx *ctx, uint32_t argc, char **argv, struct ddb_cmd_info *info)
 {
@@ -1132,6 +1183,11 @@ ddb_parse_cmd_args(struct ddb_ctx *ctx, uint32_t argc, char **argv, struct ddb_c
 		return dtx_aggr_option_parse(ctx, &info->dci_cmd_option.dci_dtx_aggr, argc, argv);
 	}
 
+	if (same(cmd, COMMAND_NAME_CSUM_DUMP)) {
+		info->dci_cmd = DDB_CMD_CSUM_DUMP;
+		return csum_dump_option_parse(ctx, &info->dci_cmd_option.dci_csum_dump, argc, argv);
+	}
+
 	ddb_errorf(ctx,
 		   "'%s' is not a valid command. Available commands are:"
 		   "'help', "
@@ -1160,7 +1216,8 @@ ddb_parse_cmd_args(struct ddb_ctx *ctx, uint32_t argc, char **argv, struct ddb_c
 		   "'dev_replace', "
 		   "'dtx_stat', "
 		   "'prov_mem', "
-		   "'dtx_aggr'\n",
+		   "'dtx_aggr', "
+		   "'csum_dump'\n",
 		   cmd);
 
 	return -DER_INVAL;
@@ -1348,6 +1405,10 @@ ddb_run_cmd(struct ddb_ctx *ctx, const char *cmd_str)
 		rc = ddb_run_dtx_aggr(ctx, &info.dci_cmd_option.dci_dtx_aggr);
 		break;
 
+	case DDB_CMD_CSUM_DUMP:
+		rc = ddb_run_csum_dump(ctx, &info.dci_cmd_option.dci_csum_dump);
+		break;
+
 	case DDB_CMD_UNKNOWN:
 		ddb_error(ctx, "Unknown command\n");
 		rc = -DER_INVAL;
@@ -1579,7 +1640,7 @@ ddb_commands_help(struct ddb_ctx *ctx)
 	ddb_print(ctx, "\n");
 
 	/* Command: dtx_aggr */
-	ddb_print(ctx, "dtx_aggr [path]\n");
+	ddb_print(ctx, "dtx_aggr [Options] [path]\n");
 	ddb_print(ctx, "\tAggregate DTX entries until a given timestamp or duration.\n");
 	ddb_print(ctx, "    [path]\n");
 	ddb_print(ctx, "\tOptional, VOS tree path of a container to aggregate.\n");
@@ -1589,6 +1650,16 @@ ddb_commands_help(struct ddb_ctx *ctx)
 	ddb_print(ctx, "    -d, --cmt_date\n");
 	ddb_print(ctx, "\tMax aggregation commit date (format '1970-01-01 00:00:00')\n");
 	ddb_print(ctx, "\n");
+
+	/* Command: csum_dump */
+	ddb_print(ctx, "csum_dump [Options] <path>\n");
+	ddb_print(ctx, "\tDump visible checksum(s) information until a given epoch.\n");
+	ddb_print(ctx, "    [path]\n");
+	ddb_print(ctx, "\tComplete VOS tree path to a Single Value or to an Array Extent.\n");
+	ddb_print(ctx, "Options:\n");
+	ddb_print(ctx, "    -e, --epoch\n");
+	ddb_print(ctx, "\tMax epoch to dump checksum(s) information (default MAX_EPOCH).\n");
+	ddb_print(ctx, "\n");
 }
 
 void
@@ -1661,4 +1732,5 @@ ddb_program_help(struct ddb_ctx *ctx)
 	ddb_print(ctx, "   dtx_stat	     Stat on DTX entries\n");
 	ddb_print(ctx, "   prov_mem	     Prepare memory environment for md-on-ssd mode.\n");
 	ddb_print(ctx, "   dtx_aggr	     Aggregate DTX entries\n");
+	ddb_print(ctx, "   csum_dump	     Dump checksum information\n");
 }

src/utils/ddb/ddb.h

@@ -126,6 +126,7 @@ enum ddb_cmd {
 	DDB_CMD_DTX_STAT                = 26,
 	DDB_CMD_PROV_MEM                = 27,
 	DDB_CMD_DTX_AGGR                = 28,
+	DDB_CMD_CSUM_DUMP               = 29,
 };
 
 /* option and argument structures for commands that need them */
@@ -236,6 +237,12 @@ struct dtx_aggr_options {
 	char                *cmt_date;
 };
 
+struct csum_dump_options {
+	char *path;
+	char *dst;
+	daos_epoch_t epoch;
+};
+
 struct ddb_cmd_info {
 	enum ddb_cmd dci_cmd;
 	union {
@@ -259,6 +266,7 @@ struct ddb_cmd_info {
 		struct dtx_stat_options      dci_dtx_stat;
 		struct prov_mem_options      dci_prov_mem;
 		struct dtx_aggr_options      dci_dtx_aggr;
+		struct csum_dump_options     dci_csum_dump;
 	} dci_cmd_option;
 };
 
@@ -330,6 +338,8 @@ int
 ddb_run_prov_mem(struct ddb_ctx *ctx, struct prov_mem_options *opt);
 int
 ddb_run_dtx_aggr(struct ddb_ctx *ctx, struct dtx_aggr_options *opt);
+int
+ddb_run_csum_dump(struct ddb_ctx *ctx, struct csum_dump_options *opt);
 
 void
 ddb_program_help(struct ddb_ctx *ctx);