db.c: Remove limit on family/key length

Consumers like media_cache have been running into issues with the previous astdb "/family/key" limit of 253 bytes when needing to store things like long URIs. An Amazon S3 URI is a good example of this. Now, instead of using a static 256 byte buffer for "/family/key", we use ast_asprintf() to dynamically create it. Both test_db.c and test_media_cache.c were also updated to use keys/URIs over the old 253 character limit. Resolves: #881 UserNote: The `ast_db_*()` APIs have had the 253 byte limit on "/family/key" removed and will now accept families and keys with a total length of up to SQLITE_MAX_LENGTH (currently 1e9!). This affects the `DB*` dialplan applications, dialplan functions, manager actions and `databse` CLI commands. Since the media_cache also uses the `ast_db_*()` APIs, you can now store resources with URIs longer than 253 bytes.
2025-09-04 11:58:52 +00:00 · 2024-09-11 10:06:17 -06:00
parent 24d641f40a
commit cc06c95ed2
3 changed files with 146 additions and 104 deletions
--- a/main/db.c
+++ b/main/db.c
@@ -116,7 +116,6 @@
 	</manager>
 ***/

-#define MAX_DB_FIELD 256
 AST_MUTEX_DEFINE_STATIC(dblock);
 static ast_cond_t dbcond;
 static sqlite3 *astdb;
@@ -138,7 +137,7 @@ DEFINE_SQL_STATEMENT(deltree_all_stmt, "DELETE FROM astdb")
 DEFINE_SQL_STATEMENT(gettree_stmt, "SELECT key, value FROM astdb WHERE key || '/' LIKE ? || '/' || '%' ORDER BY key")
 DEFINE_SQL_STATEMENT(gettree_all_stmt, "SELECT key, value FROM astdb ORDER BY key")
 DEFINE_SQL_STATEMENT(showkey_stmt, "SELECT key, value FROM astdb WHERE key LIKE '%' || '/' || ? ORDER BY key")
-DEFINE_SQL_STATEMENT(create_astdb_stmt, "CREATE TABLE IF NOT EXISTS astdb(key VARCHAR(256), value VARCHAR(256), PRIMARY KEY(key))")
+DEFINE_SQL_STATEMENT(create_astdb_stmt, "CREATE TABLE IF NOT EXISTS astdb(key TEXT, value TEXT, PRIMARY KEY(key))")

 /* This query begs an explanation:
 *
@@ -341,17 +340,17 @@ static int ast_db_rollback_transaction(void)

 int ast_db_put(const char *family, const char *key, const char *value)
 {
-	char fullkey[MAX_DB_FIELD];
-	size_t fullkey_len;
+	char *fullkey;
+	int fullkey_len;
 	int res = 0;

-	if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
-		ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
+	fullkey_len = ast_asprintf(&fullkey, "/%s/%s", family, key);
+	if (fullkey_len < 0) {
+		ast_log(LOG_WARNING, "Unable to allocate memory for family/key '/%s/%s'\n",
+			family, key);
 		return -1;
 	}

-	fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
-
 	ast_mutex_lock(&dblock);
 	if (sqlite3_bind_text(put_stmt, 1, fullkey, fullkey_len, SQLITE_STATIC) != SQLITE_OK) {
 		ast_log(LOG_WARNING, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
@@ -363,10 +362,10 @@ int ast_db_put(const char *family, const char *key, const char *value)
 		ast_log(LOG_WARNING, "Couldn't execute statement: %s\n", sqlite3_errmsg(astdb));
 		res = -1;
 	}
-
 	sqlite3_reset(put_stmt);
 	db_sync();
 	ast_mutex_unlock(&dblock);
+	ast_free(fullkey);

 	return res;
 }
@@ -388,17 +387,17 @@ int ast_db_put(const char *family, const char *key, const char *value)
 static int db_get_common(const char *family, const char *key, char **buffer, int bufferlen)
 {
 	const unsigned char *result;
-	char fullkey[MAX_DB_FIELD];
-	size_t fullkey_len;
+	char *fullkey;
+	int fullkey_len;
 	int res = 0;

-	if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
-		ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
+	fullkey_len = ast_asprintf(&fullkey, "/%s/%s", family, key);
+	if (fullkey_len < 0) {
+		ast_log(LOG_WARNING, "Unable to allocate memory for family/key '/%s/%s'\n",
+			family, key);
 		return -1;
 	}

-	fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
-
 	ast_mutex_lock(&dblock);
 	if (sqlite3_bind_text(get_stmt, 1, fullkey, fullkey_len, SQLITE_STATIC) != SQLITE_OK) {
 		ast_log(LOG_WARNING, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
@@ -420,6 +419,7 @@ static int db_get_common(const char *family, const char *key, char **buffer, int
 	}
 	sqlite3_reset(get_stmt);
 	ast_mutex_unlock(&dblock);
+	ast_free(fullkey);

 	return res;
 }
@@ -444,13 +444,14 @@ int ast_db_get_allocated(const char *family, const char *key, char **out)
 int ast_db_exists(const char *family, const char *key)
 {
 	int result;
-	char fullkey[MAX_DB_FIELD];
-	size_t fullkey_len;
+	char *fullkey;
+	int fullkey_len;
 	int res = 0;

-	fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
-	if (fullkey_len >= sizeof(fullkey)) {
-		ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
+	fullkey_len = ast_asprintf(&fullkey, "/%s/%s", family, key);
+	if (fullkey_len < 0) {
+		ast_log(LOG_WARNING, "Unable to allocate memory for family/key '/%s/%s'\n",
+			family, key);
 		return -1;
 	}

@@ -468,6 +469,7 @@ int ast_db_exists(const char *family, const char *key)
 	}
 	sqlite3_reset(exists_stmt);
 	ast_mutex_unlock(&dblock);
+	ast_free(fullkey);

 	return res;
 }
@@ -475,17 +477,17 @@ int ast_db_exists(const char *family, const char *key)

 int ast_db_del(const char *family, const char *key)
 {
-	char fullkey[MAX_DB_FIELD];
-	size_t fullkey_len;
+	char *fullkey;
+	int fullkey_len;
 	int res = 0;

-	if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
-		ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
+	fullkey_len = ast_asprintf(&fullkey, "/%s/%s", family, key);
+	if (fullkey_len < 0) {
+		ast_log(LOG_WARNING, "Unable to allocate memory for family/key '/%s/%s'\n",
+			family, key);
 		return -1;
 	}

-	fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
-
 	ast_mutex_lock(&dblock);
 	if (sqlite3_bind_text(del_stmt, 1, fullkey, fullkey_len, SQLITE_STATIC) != SQLITE_OK) {
 		ast_log(LOG_WARNING, "Couldn't bind key to stmt: %s\n", sqlite3_errmsg(astdb));
@@ -497,24 +499,25 @@ int ast_db_del(const char *family, const char *key)
 	sqlite3_reset(del_stmt);
 	db_sync();
 	ast_mutex_unlock(&dblock);
+	ast_free(fullkey);

 	return res;
 }

 int ast_db_del2(const char *family, const char *key)
 {
-	char fullkey[MAX_DB_FIELD];
+	char *fullkey;
 	char tmp[1];
-	size_t fullkey_len;
+	int fullkey_len;
 	int mres, res = 0;

-	if (strlen(family) + strlen(key) + 2 > sizeof(fullkey) - 1) {
-		ast_log(LOG_WARNING, "Family and key length must be less than %zu bytes\n", sizeof(fullkey) - 3);
+	fullkey_len = ast_asprintf(&fullkey, "/%s/%s", family, key);
+	if (fullkey_len < 0) {
+		ast_log(LOG_WARNING, "Unable to allocate memory for family/key '/%s/%s'\n",
+			family, key);
 		return -1;
 	}

-	fullkey_len = snprintf(fullkey, sizeof(fullkey), "/%s/%s", family, key);
-
 	ast_mutex_lock(&dblock);
 	if (ast_db_get(family, key, tmp, sizeof(tmp))) {
 		ast_log(LOG_WARNING, "AstDB key %s does not exist\n", fullkey);
@@ -529,31 +532,58 @@ int ast_db_del2(const char *family, const char *key)
 	sqlite3_reset(del_stmt);
 	db_sync();
 	ast_mutex_unlock(&dblock);
+	ast_free(fullkey);

 	return res;
 }

+static char *create_prefix(const char *family, const char *keytree,
+	int *prefix_len)
+{
+	char *prefix = NULL;
+	*prefix_len = 0;
+
+	if (!ast_strlen_zero(family)) {
+		if (!ast_strlen_zero(keytree)) {
+			/* Family and key tree */
+			*prefix_len = ast_asprintf(&prefix, "/%s/%s", family, keytree);
+		} else {
+			/* Family only */
+			*prefix_len = ast_asprintf(&prefix, "/%s", family);
+		}
+		if (*prefix_len < 0) {
+			ast_log(LOG_WARNING, "Unable to allocate memory for family/keytree '/%s%s%s'\n",
+				S_OR(family, ""), S_COR(keytree, "/", ""), S_OR(keytree, ""));
+			return NULL;
+		}
+	} else {
+		prefix = ast_strdup("");
+	}
+	return prefix;
+}
+
 int ast_db_deltree(const char *family, const char *keytree)
 {
 	sqlite3_stmt *stmt = deltree_stmt;
-	char prefix[MAX_DB_FIELD];
+	char *prefix = NULL;
+	int prefix_len = 0;
 	int res = 0;

-	if (!ast_strlen_zero(family)) {
-		if (!ast_strlen_zero(keytree)) {
-			/* Family and key tree */
-			snprintf(prefix, sizeof(prefix), "/%s/%s", family, keytree);
-		} else {
-			/* Family only */
-			snprintf(prefix, sizeof(prefix), "/%s", family);
-		}
-	} else {
-		prefix[0] = '\0';
+	if (ast_strlen_zero(family) && !ast_strlen_zero(keytree)) {
+		ast_log(LOG_WARNING, "Key tree '%s' specified without family\n", keytree);
+		return -1;
+	}
+
+	prefix = create_prefix(family, keytree, &prefix_len);
+	if (!prefix) {
+		return -1;
+	}
+	if (prefix_len == 0) {
 		stmt = deltree_all_stmt;
 	}

 	ast_mutex_lock(&dblock);
-	if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, -1, SQLITE_STATIC) != SQLITE_OK)) {
+	if (prefix_len && (sqlite3_bind_text(stmt, 1, prefix, prefix_len, SQLITE_STATIC) != SQLITE_OK)) {
 		ast_log(LOG_WARNING, "Couldn't bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
 		res = -1;
 	} else if (sqlite3_step(stmt) != SQLITE_DONE) {
@@ -564,6 +594,7 @@ int ast_db_deltree(const char *family, const char *keytree)
 	sqlite3_reset(stmt);
 	db_sync();
 	ast_mutex_unlock(&dblock);
+	ast_free(prefix);

 	return res;
 }
@@ -609,67 +640,60 @@ static struct ast_db_entry *db_gettree_common(sqlite3_stmt *stmt)

 struct ast_db_entry *ast_db_gettree(const char *family, const char *keytree)
 {
-	char prefix[MAX_DB_FIELD];
+	char *prefix = NULL;
+	int prefix_len = 0;
 	sqlite3_stmt *stmt = gettree_stmt;
-	size_t res = 0;
 	struct ast_db_entry *ret;

-	if (!ast_strlen_zero(family)) {
-		if (!ast_strlen_zero(keytree)) {
-			/* Family and key tree */
-			res = snprintf(prefix, sizeof(prefix), "/%s/%s", family, keytree);
-		} else {
-			/* Family only */
-			res = snprintf(prefix, sizeof(prefix), "/%s", family);
-		}
-
-		if (res >= sizeof(prefix)) {
-			ast_log(LOG_WARNING, "Requested prefix is too long: %s\n", keytree);
-			return NULL;
-		}
-	} else {
-		prefix[0] = '\0';
+	prefix = create_prefix(family, keytree, &prefix_len);
+	if (!prefix) {
+		return NULL;
+	}
+	if (prefix_len == 0) {
 		stmt = gettree_all_stmt;
 	}

 	ast_mutex_lock(&dblock);
-	if (res && (sqlite3_bind_text(stmt, 1, prefix, res, SQLITE_STATIC) != SQLITE_OK)) {
+	if (prefix_len && (sqlite3_bind_text(stmt, 1, prefix, prefix_len, SQLITE_STATIC) != SQLITE_OK)) {
 		ast_log(LOG_WARNING, "Could not bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
 		sqlite3_reset(stmt);
 		ast_mutex_unlock(&dblock);
+		ast_free(prefix);
 		return NULL;
 	}

 	ret = db_gettree_common(stmt);
 	sqlite3_reset(stmt);
 	ast_mutex_unlock(&dblock);
+	ast_free(prefix);

 	return ret;
 }

 struct ast_db_entry *ast_db_gettree_by_prefix(const char *family, const char *key_prefix)
 {
-	char prefix[MAX_DB_FIELD];
-	size_t res;
+	char *prefix = NULL;
+	int prefix_len = 0;
 	struct ast_db_entry *ret;

-	res = snprintf(prefix, sizeof(prefix), "/%s/%s", family, key_prefix);
-	if (res >= sizeof(prefix)) {
-		ast_log(LOG_WARNING, "Requested key prefix is too long: %s\n", key_prefix);
+	prefix = create_prefix(family, key_prefix, &prefix_len);
+	if (!prefix) {
 		return NULL;
 	}

 	ast_mutex_lock(&dblock);
-	if (sqlite3_bind_text(gettree_prefix_stmt, 1, prefix, res, SQLITE_STATIC) != SQLITE_OK) {
+	if (sqlite3_bind_text(gettree_prefix_stmt, 1, prefix, prefix_len, SQLITE_STATIC) != SQLITE_OK) {
 		ast_log(LOG_WARNING, "Could not bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
 		sqlite3_reset(gettree_prefix_stmt);
 		ast_mutex_unlock(&dblock);
+		ast_free(prefix);
 		return NULL;
 	}

 	ret = db_gettree_common(gettree_prefix_stmt);
 	sqlite3_reset(gettree_prefix_stmt);
 	ast_mutex_unlock(&dblock);
+	ast_free(prefix);

 	return ret;
 }
@@ -714,7 +738,7 @@ static char *handle_cli_database_put(struct ast_cli_entry *e, int cmd, struct as
 static char *handle_cli_database_get(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
 {
 	int res;
-	char tmp[MAX_DB_FIELD];
+	char *tmp = NULL;

 	switch (cmd) {
 	case CLI_INIT:
@@ -730,12 +754,14 @@ static char *handle_cli_database_get(struct ast_cli_entry *e, int cmd, struct as

 	if (a->argc != 4)
 		return CLI_SHOWUSAGE;
-	res = ast_db_get(a->argv[2], a->argv[3], tmp, sizeof(tmp));
+	res = ast_db_get_allocated(a->argv[2], a->argv[3], &tmp);
 	if (res) {
 		ast_cli(a->fd, "Database entry not found.\n");
 	} else {
 		ast_cli(a->fd, "Value: %s\n", tmp);
+		ast_free(tmp);
 	}
+
 	return CLI_SUCCESS;
 }

@@ -803,7 +829,10 @@ static char *handle_cli_database_deltree(struct ast_cli_entry *e, int cmd, struc

 static char *handle_cli_database_show(struct ast_cli_entry *e, int cmd, struct ast_cli_args *a)
 {
-	char prefix[MAX_DB_FIELD];
+	char *prefix = NULL;
+	int prefix_len = 0;
+	const char *family = a->argc > 2 ? a->argv[2] : "";
+	const char *keytree = a->argc > 3 ? a->argv[3] : "";
 	int counter = 0;
 	sqlite3_stmt *stmt = gettree_stmt;

@@ -821,26 +850,24 @@ static char *handle_cli_database_show(struct ast_cli_entry *e, int cmd, struct a
 		return NULL;
 	}

-	if (a->argc == 4) {
-		/* Family and key tree */
-		snprintf(prefix, sizeof(prefix), "/%s/%s", a->argv[2], a->argv[3]);
-	} else if (a->argc == 3) {
-		/* Family only */
-		snprintf(prefix, sizeof(prefix), "/%s", a->argv[2]);
-	} else if (a->argc == 2) {
-		/* Neither */
-		prefix[0] = '\0';
-		stmt = gettree_all_stmt;
-
-	} else {
+	if (a->argc > 4) {
 		return CLI_SHOWUSAGE;
 	}

+	prefix = create_prefix(family, keytree, &prefix_len);
+	if (!prefix) {
+		return NULL;
+	}
+	if (prefix_len == 0) {
+		stmt = gettree_all_stmt;
+	}
+
 	ast_mutex_lock(&dblock);
-	if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, -1, SQLITE_STATIC) != SQLITE_OK)) {
+	if (prefix_len && (sqlite3_bind_text(stmt, 1, prefix, prefix_len, SQLITE_STATIC) != SQLITE_OK)) {
 		ast_log(LOG_WARNING, "Couldn't bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
 		sqlite3_reset(stmt);
 		ast_mutex_unlock(&dblock);
+		ast_free(prefix);
 		return NULL;
 	}

@@ -860,6 +887,7 @@ static char *handle_cli_database_show(struct ast_cli_entry *e, int cmd, struct a

 	sqlite3_reset(stmt);
 	ast_mutex_unlock(&dblock);
+	ast_free(prefix);

 	ast_cli(a->fd, "%d results found.\n", counter);
 	return CLI_SUCCESS;
@@ -990,7 +1018,7 @@ static int manager_dbget(struct mansession *s, const struct message *m)
 	char idText[256];
 	const char *family = astman_get_header(m, "Family");
 	const char *key = astman_get_header(m, "Key");
-	char tmp[MAX_DB_FIELD];
+	char *tmp = NULL;
 	int res;

 	if (ast_strlen_zero(family)) {
@@ -1006,7 +1034,7 @@ static int manager_dbget(struct mansession *s, const struct message *m)
 	if (!ast_strlen_zero(id))
 		snprintf(idText, sizeof(idText) ,"ActionID: %s\r\n", id);

-	res = ast_db_get(family, key, tmp, sizeof(tmp));
+	res = ast_db_get_allocated(family, key, &tmp);
 	if (res) {
 		astman_send_error(s, m, "Database entry not found");
 	} else {
@@ -1019,7 +1047,7 @@ static int manager_dbget(struct mansession *s, const struct message *m)
 				"%s"
 				"\r\n",
 				family, key, tmp, idText);
-
+		ast_free(tmp);
 		astman_send_list_complete_start(s, m, "DBGetComplete", 1);
 		astman_send_list_complete_end(s);
 	}
@@ -1028,7 +1056,8 @@ static int manager_dbget(struct mansession *s, const struct message *m)

 static int manager_db_tree_get(struct mansession *s, const struct message *m)
 {
-	char prefix[MAX_DB_FIELD];
+	char *prefix;
+	int prefix_len = 0;
 	char idText[256];
 	const char *id = astman_get_header(m,"ActionID");
 	const char *family = astman_get_header(m, "Family");
@@ -1036,15 +1065,12 @@ static int manager_db_tree_get(struct mansession *s, const struct message *m)
 	sqlite3_stmt *stmt = gettree_stmt;
 	int count = 0;

-	if (!ast_strlen_zero(family) && !ast_strlen_zero(key)) {
-		/* Family and key tree */
-		snprintf(prefix, sizeof(prefix), "/%s/%s", family, key);
-	} else if (!ast_strlen_zero(family)) {
-		/* Family only */
-		snprintf(prefix, sizeof(prefix), "/%s", family);
-	} else {
-		/* Neither */
-		prefix[0] = '\0';
+	prefix = create_prefix(family, key, &prefix_len);
+	if (!prefix) {
+		astman_send_error(s, m, "Unable to allocate memory for Family/Key");
+		return 0;
+	}
+	if (prefix_len == 0) {
 		stmt = gettree_all_stmt;
 	}

@@ -1054,10 +1080,11 @@ static int manager_db_tree_get(struct mansession *s, const struct message *m)
 	}

 	ast_mutex_lock(&dblock);
-	if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, -1, SQLITE_STATIC) != SQLITE_OK)) {
+	if (!ast_strlen_zero(prefix) && (sqlite3_bind_text(stmt, 1, prefix, prefix_len, SQLITE_STATIC) != SQLITE_OK)) {
 		ast_log(LOG_WARNING, "Couldn't bind %s to stmt: %s\n", prefix, sqlite3_errmsg(astdb));
 		sqlite3_reset(stmt);
 		ast_mutex_unlock(&dblock);
+		ast_free(prefix);
 		astman_send_error(s, m, "Unable to search database");
 		return 0;
 	}
@@ -1085,6 +1112,7 @@ static int manager_db_tree_get(struct mansession *s, const struct message *m)

 	sqlite3_reset(stmt);
 	ast_mutex_unlock(&dblock);
+	ast_free(prefix);

 	astman_send_list_complete_start(s, m, "DBGetTreeComplete", count);
 	astman_send_list_complete_end(s);