NCS/age_cache_simulation
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(optimal ttl): Added calculation for optimal ttl of each object

Browse Source 
Signed-off-by: Tuan-Dat Tran <tuan-dat.tran@tudattr.dev>
This commit is contained in:
Tuan-Dat Tran
2024-12-03 15:15:56 +01:00
parent 272f722f23
commit 0ea1fb5d07
2 changed files with 697 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
01_nb_cncf_optimization/02-objective_multi-core_gridsearch.ipynb
View File

@@ -40,9 +40,6 @@
"def eta_star(db_object_count, c_f, cache_sz, c_delta, lambda_vals):\n", "def eta_star(db_object_count, c_f, cache_sz, c_delta, lambda_vals):\n",
" num = (db_object_count * c_f - cache_sz * c_delta)\n", " num = (db_object_count * c_f - cache_sz * c_delta)\n",
" denom = np.sum(1.0/lambda_vals)\n", " denom = np.sum(1.0/lambda_vals)\n",
" if denom == 0:\n",
" print(\"sum(1.0/lambda_vals) == 0\")\n",
" print(db_object_count, c_f, cache_sz, c_delta, lambda_vals)\n",
" return max(0, num/denom)" " return max(0, num/denom)"
] ]
}, },
@@ -160,12 +157,29 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 9, "execution_count": 9,
"id": "a289bb1a-0385-4835-bc92-88304c1834df",
"metadata": {},
"outputs": [],
"source": [
"def optimize_ttl(optimized_hitrates, lambda_vals):\n",
" ti_values = np.where(\n",
" optimized_hitrates < 1,\n",
" -1 / lambda_vals * np.log(1 - optimized_hitrates),\n",
" np.inf\n",
" )\n",
" \n",
" return ti_values"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "bd4536e9-273b-4f49-b06c-2f00605e0f7d", "id": "bd4536e9-273b-4f49-b06c-2f00605e0f7d",
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"# Define the task to be parallelized\n", "# Define the task to be parallelized\n",
"def grid_search_task(db_object_count, cache_size, c_f, c_delta, db_object_counts, cache_sizes, c_f_values, c_delta_values):\n", "def grid_search_task(db_object_count, cache_size, c_f, c_delta):\n",
" if db_object_count < cache_size:\n", " if db_object_count < cache_size:\n",
" return None # Skip this combination if db_object_count < cache_size\n", " return None # Skip this combination if db_object_count < cache_size\n",
" \n", " \n",
@@ -175,10 +189,12 @@
" # Call the optimization function\n", " # Call the optimization function\n",
" optimized_hitrates = optimize_hitrates(db_object_count, cache_size, c_f, c_delta, lambda_vals)\n", " optimized_hitrates = optimize_hitrates(db_object_count, cache_size, c_f, c_delta, lambda_vals)\n",
"\n", "\n",
" optimized_ttl = optimize_ttl(optimized_hitrates, lambda_vals)\n",
" \n",
" # Compute the objective function\n", " # Compute the objective function\n",
" objective = objective_function(optimized_hitrates, c_f, c_delta, lambda_vals)\n", " objective = objective_function(optimized_hitrates, c_f, c_delta, lambda_vals)\n",
"\n", "\n",
" return (objective, db_object_count, cache_size, c_f, c_delta)\n", " return (objective, optimized_ttl, db_object_count, cache_size, c_f, c_delta)\n",
"\n", "\n",
"# Perform grid search with parallelization and tqdm progress bar\n", "# Perform grid search with parallelization and tqdm progress bar\n",
"def grid_search(db_object_counts, cache_sizes, c_f_values, c_delta_values):\n", "def grid_search(db_object_counts, cache_sizes, c_f_values, c_delta_values):\n",
@@ -187,7 +203,7 @@
" \n", " \n",
" # Use Parallel from joblib to parallelize the grid search\n", " # Use Parallel from joblib to parallelize the grid search\n",
" task_results = Parallel(n_jobs=-1, verbose=1)(\n", " task_results = Parallel(n_jobs=-1, verbose=1)(\n",
" delayed(grid_search_task)(db_object_count, cache_size, c_f, c_delta, db_object_counts, cache_sizes, c_f_values, c_delta_values)\n", " delayed(grid_search_task)(db_object_count, cache_size, c_f, c_delta)\n",
" for db_object_count, cache_size, c_f, c_delta in itertools.product(db_object_counts, cache_sizes, c_f_values, c_delta_values)\n", " for db_object_count, cache_size, c_f, c_delta in itertools.product(db_object_counts, cache_sizes, c_f_values, c_delta_values)\n",
" )\n", " )\n",
"\n", "\n",
@@ -197,23 +213,155 @@
" results.append(result)\n", " results.append(result)\n",
" \n", " \n",
" # Convert the results into a pandas DataFrame\n", " # Convert the results into a pandas DataFrame\n",
" df = pd.DataFrame(results, columns=[\"Objective\", \"db_object_count\", \"cache_size\", \"c_f (Miss Cost)\", \"c_delta (Refresh Cost)\"])\n", " df = pd.DataFrame(results, columns=[\"Objective\", \"Optimal TTL\", \"db_object_count\", \"cache_size\", \"c_f (Miss Cost)\", \"c_delta (Refresh Cost)\"])\n",
" \n", " \n",
" return df\n" " return df\n"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": 11, "execution_count": null,
"id": "a92c6772-6609-41a8-a3d1-4d640b69a864", "id": "a92c6772-6609-41a8-a3d1-4d640b69a864",
"metadata": {}, "metadata": {},
"outputs": [ "outputs": [
{ {
"name": "stdout", "name": "stderr",
"output_type": "stream", "output_type": "stream",
"text": [ "text": [
"CPU times: user 7.96 ms, sys: 21.6 ms, total: 29.6 ms\n", "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 12 concurrent workers.\n",
"Wall time: 27 ms\n" "/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"[Parallel(n_jobs=-1)]: Done 26 tasks | elapsed: 0.6s\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"[Parallel(n_jobs=-1)]: Done 1420 tasks | elapsed: 1.3s\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n",
"/tmp/ipykernel_137117/3256270718.py:4: RuntimeWarning: divide by zero encountered in log\n"
] ]
} }
], ],

538
01_nb_cncf_optimization/03-plot_objective_optimization.ipynb Normal file
View File

File diff suppressed because one or more lines are too long