{ "cells": [ { "cell_type": "code", "execution_count": 1, "id": "ab5cd7d1-1a57-46fc-8282-dae0a6cc2944", "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import random\n", "import pandas as pd" ] }, { "cell_type": "code", "execution_count": 2, "id": "3d1ad0b9-f6a8-4e98-84aa-6e02e4279954", "metadata": {}, "outputs": [], "source": [ "DATABASE_OBJECT_COUNT = 100\n", "CACHE_SIZE = DATABASE_OBJECT_COUNT/2\n", "ZIPF_CONSTANT = 2\n", "\n", "CACHE_MISS_COST = 2\n", "CACHE_REFRESH_COST = 1\n", "\n", "SEED = 42\n", "np.random.seed(SEED)\n", "random.seed(SEED)\n", "\n", "LAMBDA_VALUES = np.array([np.random.zipf(ZIPF_CONSTANT) for i in np.arange(1, DATABASE_OBJECT_COUNT + 1,1)])" ] }, { "cell_type": "code", "execution_count": 3, "id": "9cc83cf6-5c78-4f0d-b7cb-08cdb80c362e", "metadata": {}, "outputs": [], "source": [ "# LAMBDA_VALUES = np.array([0.03, 0.04,0.05,0.06,0.07,1,1.1,1.2,1.3,1.4,1.5])\n", "# DATABASE_OBJECT_COUNT = len(LAMBDA_VALUES)\n", "# CACHE_SIZE = 4.4\n", "# CACHE_MISS_COST = 7\n", "# CACHE_REFRESH_COST = 1" ] }, { "cell_type": "code", "execution_count": 4, "id": "3dc07233-0b56-4fee-a93b-212836c18b42", "metadata": {}, "outputs": [], "source": [ "db_object_count = DATABASE_OBJECT_COUNT\n", "cache_sz = CACHE_SIZE\n", "\n", "lambda_vals = LAMBDA_VALUES\n", "c_f = CACHE_MISS_COST\n", "c_delta = CACHE_REFRESH_COST" ] }, { "cell_type": "code", "execution_count": 5, "id": "5a27d416-8f98-4814-af9e-6c6bef95f4ef", "metadata": {}, "outputs": [], "source": [ "def eta_star(db_object_count, c_f, cache_sz, c_delta, lambda_vals):\n", " num = (db_object_count * c_f - cache_sz * c_delta)\n", " denom = np.sum(1.0/lambda_vals)\n", " return max(0, num/denom)" ] }, { "cell_type": "code", "execution_count": 6, "id": "6276a9ce-f839-4fe6-90f2-2195cf065fc8", "metadata": {}, "outputs": [], "source": [ "def h_i_star(c_f, eta, lambda_vals, c_delta):\n", " optimized_hitrate = (c_f - (eta/lambda_vals)) / c_delta\n", " return optimized_hitrate" ] }, { "cell_type": "code", "execution_count": 7, "id": "dcd31a8c-6864-4b9a-8bb3-998f0c32baf6", "metadata": {}, "outputs": [], "source": [ "def get_index_of_furthest_hitrate_from_boundary(hitrates):\n", " lower_bound_violation = hitrates[(hitrates < 0)]\n", " upper_bound_violation = hitrates[(hitrates > 1)]\n", " smallest_delta = np.abs(np.min(lower_bound_violation))\n", " biggest_delta = np.max(upper_bound_violation) - 1\n", " if smallest_delta > biggest_delta:\n", " print(smallest_delta)\n", " index = np.where(hitrates == np.min(local_hitrates))[0][0]\n", " return index\n", " else:\n", " \n", " index = np.where(hitrates == np.max(local_hitrates))[0][0]\n", " return index" ] }, { "cell_type": "code", "execution_count": 8, "id": "9d774304-ae68-43b3-a76a-e970c06c5236", "metadata": {}, "outputs": [], "source": [ "def get_index_of_furthest_hitrate_from_boundary(hitrates):\n", " outside_bounds = (hitrates < 0) | (hitrates > 1)\n", " distances = np.where(outside_bounds, np.maximum(np.abs(hitrates - 0), np.abs(hitrates - 1)), -np.inf)\n", " index = np.argmax(distances)\n", " return index" ] }, { "cell_type": "code", "execution_count": 9, "id": "19678083-15e1-439b-be8c-42033d501644", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([ 1, 3, 1, 1, 2, 1, 5, 1, 1, 1, 2, 1, 1, 1, 2, 2, 1,\n", " 1, 3, 1, 1, 1, 1, 2, 1, 1, 1, 5, 1, 1, 1, 4, 1, 4,\n", " 1, 1, 1, 3, 8, 1, 4, 4, 2, 1, 1, 1, 10, 1, 1, 1, 5,\n", " 9, 1, 1, 1, 1, 1, 17, 2, 1, 26, 1, 1, 2, 1, 10, 1, 69,\n", " 1, 1, 2, 1, 1, 1, 3, 2, 2, 3, 15, 1, 1, 5, 2, 1, 1,\n", " 2, 1, 2, 1, 1, 2, 2, 3, 1, 2, 1, 1, 37, 4, 2])" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "lambda_vals" ] }, { "cell_type": "code", "execution_count": 10, "id": "ccd4b95d-1cdd-4c99-a22e-4b31338993cf", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2.1159070575516945\n" ] }, { "data": { "text/plain": [ "array([-0.11590706, 1.29469765, -0.11590706, -0.11590706, 0.94204647,\n", " -0.11590706, -0.11590706, -0.11590706, -0.11590706, 0.94204647,\n", " -0.11590706, -0.11590706, -0.11590706, 0.94204647, 0.94204647,\n", " -0.11590706, -0.11590706, 1.29469765, -0.11590706, -0.11590706,\n", " -0.11590706, -0.11590706, 0.94204647, -0.11590706, -0.11590706,\n", " -0.11590706, -0.11590706, -0.11590706, -0.11590706, 1.47102324,\n", " -0.11590706, 1.47102324, -0.11590706, -0.11590706, -0.11590706,\n", " 1.29469765, 1.73551162, -0.11590706, 1.47102324, 1.47102324,\n", " 0.94204647, -0.11590706, -0.11590706, -0.11590706, 1.78840929,\n", " -0.11590706, -0.11590706, -0.11590706, 1.76489922, -0.11590706,\n", " -0.11590706, -0.11590706, -0.11590706, -0.11590706, 1.87553488,\n", " 0.94204647, -0.11590706, 1.91861896, -0.11590706, -0.11590706,\n", " 0.94204647, -0.11590706, 1.78840929, -0.11590706, 1.96933468,\n", " -0.11590706, -0.11590706, 0.94204647, -0.11590706, -0.11590706,\n", " -0.11590706, 1.29469765, 0.94204647, 0.94204647, 1.29469765,\n", " 1.85893953, -0.11590706, -0.11590706, 0.94204647, -0.11590706,\n", " -0.11590706, 0.94204647, -0.11590706, 0.94204647, -0.11590706,\n", " -0.11590706, 0.94204647, 0.94204647, 1.29469765, -0.11590706,\n", " 0.94204647, -0.11590706, -0.11590706, 1.94281332, 1.47102324,\n", " 0.94204647])" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "eta = eta_star(db_object_count, c_f, cache_sz, c_delta, lambda_vals[lambda_vals != lambda_vals[6]])\n", "print(eta)\n", "optimized_hitrates = (c_f - eta / lambda_vals[lambda_vals != lambda_vals[6]]) / c_delta\n", "optimized_hitrates" ] }, { "cell_type": "code", "execution_count": 11, "id": "05b17074-719f-4bca-8434-2aaee26094d0", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
0
count96.000000
mean0.437500
std0.726101
min-0.115907
25%-0.115907
50%-0.115907
75%0.942046
max1.969335
\n", "
" ], "text/plain": [ " 0\n", "count 96.000000\n", "mean 0.437500\n", "std 0.726101\n", "min -0.115907\n", "25% -0.115907\n", "50% -0.115907\n", "75% 0.942046\n", "max 1.969335" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pd.DataFrame(optimized_hitrates).describe()" ] }, { "cell_type": "code", "execution_count": 12, "id": "0e21c26f-058a-4e56-a5ad-1c47bf28656c", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Optimized: 67 1.97 // [ 1.79077042 -0.09229584 1. -0.09229584 -0.09229584]\n", "Optimized: 97 1.94 // [-0.07876743 -0.07876743 1. 1.48030814 0.96061628]\n", "Optimized: 60 1.92 // [ 0.96720258 -0.06559484 1. -0.06559484 -0.06559484]\n", "Optimized: 57 1.88 // [-0.05274002 -0.05274002 1. 0.97362999 -0.05274002]\n", "Optimized: 78 1.86 // [ 0.97977406 1.31984937 1. -0.04045188 -0.04045188]\n", "Optimized: 46 1.80 // [-0.02836604 -0.02836604 1. -0.02836604 -0.02836604]\n", "Optimized: 65 1.80 // [ 0.99140044 -0.01719911 1. -0.01719911 1. ]\n", "Optimized: 51 1.78 // [-0.00600086 1.59879983 1. -0.00600086 -0.00600086]\n", "Optimized: 38 1.75 // [0.00491746 1.33497249 1. 0.00491746 1.50122936]\n", "Optimized: 6 1.60 // [1.00774103 0.01548205 1. 0.01548205 0.01548205]\n", "Optimized: 27 1.60 // [0.02399435 0.02399435 1. 0.02399435 0.02399435]\n", "Optimized: 50 1.61 // [0.03255485 0.03255485 1. 1. 0.03255485]\n", "Optimized: 81 1.61 // [0.04116395 0.04116395 1. 1.02058197 0.04116395]\n", "Optimized: 31 1.51 // [0.04982206 0.04982206 1. 0.04982206 1.51245552]\n", "Optimized: 33 1.51 // [1. 0.05714286 1. 0.05714286 0.05714286]\n", "Optimized: 40 1.52 // [1. 0.06451613 1. 1.51612903 1.03225806]\n", "Optimized: 41 1.52 // [0.07194245 1. 1. 1.03597122 0.07194245]\n", "Optimized: 98 1.52 // [0.07942238 1. 1. 1.03971119]\n", "Optimized: 1 1.36 // []\n", "Optimized: 18 1.36 // [0.09223301 0.09223301 1. 0.09223301 0.09223301]\n", "Optimized: 37 1.37 // [0.09756098 0.09756098 1. 1. 0.09756098]\n", "Optimized: 74 1.37 // [0.10294118 0.10294118 1. 1.05147059 1.05147059]\n", "Optimized: 77 1.37 // [1.05418719 1.05418719 1. 1. 0.10837438]\n", "Optimized: 92 1.37 // [1.05693069 1.05693069 1. 0.11386139 1.05693069]\n", "Optimized: 4 1.06 // [0.11940299 0.11940299 1. 0.11940299 1. ]\n", "Optimized: 10 1.06 // [0.12030075 0.12030075 1. 0.12030075 0.12030075]\n", "Optimized: 14 1.06 // [0.12121212 0.12121212 1. 1.06060606 0.12121212]\n", "Optimized: 15 1.06 // [0.1221374 1. 1. 0.1221374 0.1221374]\n", "Optimized: 23 1.06 // [0.12307692 0.12307692 1. 0.12307692 0.12307692]\n", "Optimized: 42 1.06 // [1. 1. 1. 0.12403101 0.12403101]\n", "Optimized: 58 1.06 // [0.125 1. 1. 0.125 1. ]\n", "Optimized: 63 1.06 // [0.12598425 0.12598425 1. 0.12598425 1. ]\n", "Optimized: 70 1.06 // [0.12698413 0.12698413 1. 0.12698413 0.12698413]\n", "Optimized: 75 1.06 // [0.128 1. 1. 1.064 1. ]\n", "Optimized: 76 1.06 // [1. 1. 1. 1. 1.]\n", "Optimized: 82 1.07 // [0.1300813 1. 1. 0.1300813 0.1300813]\n", "Optimized: 85 1.07 // [0.13114754 0.13114754 1. 0.13114754 1.06557377]\n", "Optimized: 87 1.07 // [1. 0.1322314 1. 0.1322314 0.1322314]\n", "Optimized: 90 1.07 // [0.13333333 0.13333333 1. 1.06666667 1. ]\n", "Optimized: 91 1.07 // [0.13445378 1. 1. 1. 0.13445378]\n", "Optimized: 94 1.07 // [1. 0.13559322 1. 0.13559322 0.13559322]\n", "Optimized: 99 1.07 // [1. 1. 1.]\n", "All values optimized.\n" ] } ], "source": [ "\"\"\"\n", "Perform theoretical optimization to compute optimal hit probabilities.\n", "\n", "Parameters:\n", "- lambda_vals (numpy array): Request rates for each item.\n", "- B (float): Total cache size.\n", "- c_f (float): Fetching linear cost (cache miss cost).\n", "- c_delta (float): Age linear cost.\n", "\n", "Returns:\n", "- h_opt (numpy array): Optimal hit probabilities for each item.\n", "\"\"\"\n", "optimized_hitrates = np.zeros(DATABASE_OBJECT_COUNT)\n", "current_db_object_count = DATABASE_OBJECT_COUNT\n", "current_cache_size = CACHE_SIZE\n", "\n", "differenc_set = np.arange(DATABASE_OBJECT_COUNT)\n", "fix_i = []\n", "\n", "while True:\n", " if current_db_object_count == 0:\n", " print(\"No objects left to optimize.\")\n", " if current_cache_size > 0:\n", " print(\"Add obj with optimized hitrate 0 and add them to optimization pool for re-optimization.\")\n", " # Redistribute unused cache size among items with zero hit probability\n", " differenc_set = np.where(optimized_hitrates == 0)[0]\n", " fix_i = np.setdiff1d(np.arange(DATABASE_OBJECT_COUNT), differenc_set).tolist()\n", " current_db_object_count = len(differenc_set)\n", " continue\n", " else:\n", " \"Reset\"\n", " optimized_hitrates[differenc_set] = 0\n", " break\n", " # Compute Lagrangian multiplier and optimal hit probabilities\n", " eta = eta_star(current_db_object_count, c_f, current_cache_size, c_delta, lambda_vals[differenc_set])\n", " optimized_hitrates[differenc_set] = (c_f - eta / lambda_vals[differenc_set]) / c_delta\n", " if eta < 0:\n", " print(\"eta was negative.\")\n", " current_cache_size = current_db_object_count * c_f / c_delta # Adjust cache size for next iteration\n", " continue\n", " \n", " if len((optimized_hitrates[differenc_set])[((optimized_hitrates[differenc_set]) < 0) | ((optimized_hitrates[differenc_set])> 1)]) == 0:\n", " print(\"All values optimized.\")\n", " break\n", " \n", " max_outbound_index = get_index_of_furthest_hitrate_from_boundary(optimized_hitrates)\n", " fix_i.append(max_outbound_index)\n", " differenc_set = np.setdiff1d(np.arange(DATABASE_OBJECT_COUNT), fix_i)\n", "\n", " old_hitrate = optimized_hitrates[max_outbound_index]\n", " optimized_hitrates[max_outbound_index] = (1 if optimized_hitrates[max_outbound_index] > 1 else 0)\n", " \n", " print(f\"Optimized: {max_outbound_index} {old_hitrate:.2f} // {optimized_hitrates[max_outbound_index-2:max_outbound_index+3]}\")\n", " \n", " current_db_object_count -= 1\n", " current_cache_size -= optimized_hitrates[max_outbound_index]" ] }, { "cell_type": "code", "execution_count": 13, "id": "f559ee7a-be2f-4076-b01c-f08950ad5a88", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([0.13793103, 1. , 0.13793103, 0.13793103, 1. ,\n", " 0.13793103, 1. , 0.13793103, 0.13793103, 0.13793103,\n", " 1. , 0.13793103, 0.13793103, 0.13793103, 1. ,\n", " 1. , 0.13793103, 0.13793103, 1. , 0.13793103,\n", " 0.13793103, 0.13793103, 0.13793103, 1. , 0.13793103,\n", " 0.13793103, 0.13793103, 1. , 0.13793103, 0.13793103,\n", " 0.13793103, 1. , 0.13793103, 1. , 0.13793103,\n", " 0.13793103, 0.13793103, 1. , 1. , 0.13793103,\n", " 1. , 1. , 1. , 0.13793103, 0.13793103,\n", " 0.13793103, 1. , 0.13793103, 0.13793103, 0.13793103,\n", " 1. , 1. , 0.13793103, 0.13793103, 0.13793103,\n", " 0.13793103, 0.13793103, 1. , 1. , 0.13793103,\n", " 1. , 0.13793103, 0.13793103, 1. , 0.13793103,\n", " 1. , 0.13793103, 1. , 0.13793103, 0.13793103,\n", " 1. , 0.13793103, 0.13793103, 0.13793103, 1. ,\n", " 1. , 1. , 1. , 1. , 0.13793103,\n", " 0.13793103, 1. , 1. , 0.13793103, 0.13793103,\n", " 1. , 0.13793103, 1. , 0.13793103, 0.13793103,\n", " 1. , 1. , 1. , 0.13793103, 1. ,\n", " 0.13793103, 0.13793103, 1. , 1. , 1. ])" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "optimized_hitrates" ] }, { "cell_type": "code", "execution_count": 14, "id": "8b2d3cea-1cc0-476e-92bf-2ac4344a9b1b", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
0
count100.000000
mean0.500000
std0.427625
min0.137931
25%0.137931
50%0.137931
75%1.000000
max1.000000
\n", "
" ], "text/plain": [ " 0\n", "count 100.000000\n", "mean 0.500000\n", "std 0.427625\n", "min 0.137931\n", "25% 0.137931\n", "50% 0.137931\n", "75% 1.000000\n", "max 1.000000" ] }, "execution_count": 14, "metadata": {}, "output_type": "execute_result" } ], "source": [ "pd.DataFrame(optimized_hitrates).describe()" ] }, { "cell_type": "code", "execution_count": null, "id": "7a998837-72b8-4039-95a5-ca8d9c8e65ab", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "graphs", "language": "python", "name": "graphs" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.7" } }, "nbformat": 4, "nbformat_minor": 5 }