iui-group-l-name-zensiert/1-first-project/ies/Tensor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "024b33fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import os\n",
    "# os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'\n",
    "# os.environ['CUDA_VISIBLE_DEVICES'] = '2'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "72397786",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from math import isqrt\n",
    "import pickle\n",
    "from tqdm import tqdm\n",
    "delim = ';'\n",
    "\n",
    "user_count = 100\n",
    "base_path = '/opt/iui-datarelease1-sose2021/'\n",
    "Xpickle_file = './X.pickle'\n",
    "ypickle_file = './y.pickle'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5f67ba6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_pickles():\n",
    "    _p = open(Xpickle_file, 'rb')\n",
    "    X = pickle.load(_p)\n",
    "    _p.close()\n",
    "        \n",
    "    _p = open(ypickle_file, 'rb')\n",
    "    y = pickle.load(_p)\n",
    "    _p.close()\n",
    "    \n",
    "    return (np.asarray(X, dtype=pd.DataFrame), np.asarray(y, dtype=str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "805421f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "x,y = load_pickles()\n",
    "xshorted_pickle_file = './X_shorted.pickle'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a6a00ee3",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# yshorted_pickle_file = \"./y_shorted.pickle\"  wird glaube ich nicht benötigt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "877573a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def shorten(l):\n",
    "    temp        = l\n",
    "    thresh      = 100\n",
    "    temp_over_T = 0\n",
    "    isOver      = False\n",
    "    temp_short  = []\n",
    "    temp_final  = []\n",
    "    \n",
    "    for a in range (0, len(temp) ):\n",
    "        if(temp[a]>thresh):\n",
    "            temp_over_T = a\n",
    "            isOver      = True\n",
    "            break\n",
    "    \n",
    "    for x in range(temp_over_T-3, len(temp)):\n",
    "        temp_short.append(f[x])\n",
    "        \n",
    "    isOver = False\n",
    "        \n",
    "    for y in range ((len(temp_short)-1),0, -1):     \n",
    "        if(temp_short[y] > thresh):\n",
    "            temp_over_T  = y\n",
    "            isOver       = True\n",
    "            break\n",
    "        \n",
    "    for z in range(0, temp_over_T+1):\n",
    "        temp_final.append(temp_short[z])\n",
    "    \n",
    "    temp_final.append(0)\n",
    "    return temp_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93303e1e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5eb3f0c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def save_shorted(l):\n",
    "    _p = open(xshorted_pickle_file, 'wb')\n",
    "    pickle.dump(l, _p)\n",
    "    _p.close()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "26bcaaad",
   "metadata": {},
   "outputs": [],
   "source": [
    "def shorten_pickle(l):\n",
    "    temp        = l\n",
    "    thresh      = 100\n",
    "    temp_over_T = 0\n",
    "    isOver      = False\n",
    "    temp_short  = [] ##Daten nachdem vorne abgeschnitten wird\n",
    "    temp_final  = [] ##Daten nachdem auch hinten abgeschnitten wurde\n",
    "    \n",
    "    temp_X       = [] ## Zweite Dimension von Temp, beinhaltet force \n",
    "    temp_short_X = []\n",
    "    \n",
    "    for a in tqdm(range (0, len(temp))): \n",
    "         temp_X = temp[a] \n",
    "            \n",
    "        \n",
    "#         for b in range (0, len(temp_X)):    \n",
    "#             if(temp_X[b]>thresh):\n",
    "#                 temp_over_T = b\n",
    "#                 isOver      = True ##gucken ob löschen\n",
    "#                 break\n",
    "            \n",
    "        \n",
    "#         for x in range (temp_over_T,len(temp_X)):\n",
    "#             temp_short_X.append(temp_X[x]) ##hier werden die Daten von y appended in eine liste ( Von form data[x][y])\n",
    "            \n",
    "#         isOver = False\n",
    "        \n",
    "#         for y in range ((len(temp_short_X)-1),0, -1):     \n",
    "#             if(temp_short_X[y] > thresh):\n",
    "#                 temp_over_T  = y\n",
    "#                 isOver       = True\n",
    "#                 break\n",
    "        \n",
    "#         for z in range(0, temp_over_T+1):\n",
    "#             temp_short.append(temp_short_X[z]) ##hier wird [y] als einzelne Datei gespeichert\n",
    "        \n",
    "#         temp_short.append(0) # Damit beim Plot die linie bis nach unten geht\n",
    "        \n",
    "        \n",
    "#     temp_final.append(temp_short)\n",
    "#    temp_final =  shorten_v2()\n",
    "   \n",
    "    \n",
    "    \n",
    "    return temp_final\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "32584756",
   "metadata": {},
   "outputs": [],
   "source": [
    "# shorted_f = shorten_pickle(f_data)\n",
    "    \n",
    "# print(shorted_f[0])\n",
    "# print (\"Saving now...\")\n",
    "# save_shorted(shorted_f)\n",
    "\n",
    "# print (\"Saving complete\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e7056b09",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_shorted():\n",
    "    _p = open(xshorted_pickle_file, 'rb')\n",
    "    x = pickle.load(_p)\n",
    "    _p.close()\n",
    "        \n",
    "   \n",
    "    print(\"done\")\n",
    "    return (np.asarray(x, dtype=pd.DataFrame))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fad8741d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "# frame = load_shorted()\n",
    "# print (\"returned\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "96abe0b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def shorten_v2(l):\n",
    "    \n",
    "    \n",
    "    temp_X       = l['Force']\n",
    "    thresh       = 100\n",
    "    temp_over_T  = 0\n",
    "    maxVal       = 0\n",
    "    temp_short   = [] ##Daten nachdem vorne abgeschnitten wird\n",
    "    temp_final   = [] ##Daten nachdem auch hinten abgeschnitten wurde\n",
    "    temp_short_X = []\n",
    "    \n",
    "    \n",
    "    for a in range (0, len(temp_X) ):\n",
    "        if(temp_X[a]>maxVal):\n",
    "            maxVal = temp_X[a]\n",
    "            \n",
    "\n",
    "    threshVal = maxVal*0.05\n",
    "    \n",
    "        \n",
    "    for b in range (0, len(temp_X)):    \n",
    "        if(temp_X[b]>threshVal):\n",
    "            temp_over_T = b\n",
    "            break\n",
    "               \n",
    "    for x in range (temp_over_T,len(temp_X)):\n",
    "        temp_short_X.append(temp_X[x]) ##hier werden die Daten von y appended in eine liste ( Von form data[x][y])\n",
    "            \n",
    "      \n",
    "    for y in range ((len(temp_short_X)-1),0, -1):     \n",
    "         if(temp_short_X[y] > threshVal):\n",
    "            temp_over_T  = y\n",
    "            break\n",
    "        \n",
    "    for z in range(0, temp_over_T+1):\n",
    "        temp_short.append(temp_short_X[z]) ##hier wird [y] als einzelne Datei gespeichert\n",
    "        \n",
    "    temp_short.append(0) # Damit beim Plot die linie bis nach unten geht\n",
    "        \n",
    "        \n",
    "    temp_final.append(temp_short)\n",
    "    \n",
    "    return temp_final\n",
    "    \n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2ec14876",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = list(map(shorten_v2, x))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "93d7b7e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[list([362.0, 683.0, 994.0, 1306.0, 1380.0, 1526.0, 1724.0, 1895.0, 1985.0, 2077.0, 2040.0, 1997.0, 1917.0, 1623.0, 1357.0, 1087.0, 785.0, 450.0, 146.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 67.0, 615.0, 927.0, 1212.0, 1450.0, 1483.0, 1546.0, 1634.0, 1617.0, 1691.0, 1654.0, 1663.0, 1618.0, 1550.0, 1519.0, 1519.0, 1742.0, 1892.0, 1966.0, 1966.0, 2092.0, 2040.0, 2048.0, 1997.0, 1953.0, 1946.0, 1926.0, 1791.0, 1497.0, 1112.0, 678.0, 279.0, 0])]\n",
      "----------------------\n",
      "[list([656.0, 729.0, 1036.0, 1253.0, 1465.0, 1662.0, 1707.0, 1730.0, 1897.0, 1890.0, 1885.0, 1856.0, 1833.0, 1789.0, 1746.0, 1752.0, 1766.0, 1799.0, 1780.0, 1759.0, 1743.0, 1732.0, 1735.0, 1824.0, 1788.0, 1737.0, 1637.0, 1547.0, 1443.0, 1199.0, 862.0, 400.0, 0])]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/jupyterhub/lib/python3.8/site-packages/numpy/core/_asarray.py:83: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray\n",
      "  return array(a, dtype, copy=False, order=order)\n"
     ]
    }
   ],
   "source": [
    "data = np.asarray(data)\n",
    "#data.shape\n",
    "print(data[0])\n",
    "print (\"----------------------\")\n",
    "print(data[len(data)-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bdd09b3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13102, 1)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b4b048c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[list([362.0, 683.0, 994.0, 1306.0, 1380.0, 1526.0, 1724.0, 1895.0, 1985.0, 2077.0, 2040.0, 1997.0, 1917.0, 1623.0, 1357.0, 1087.0, 785.0, 450.0, 146.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 67.0, 615.0, 927.0, 1212.0, 1450.0, 1483.0, 1546.0, 1634.0, 1617.0, 1691.0, 1654.0, 1663.0, 1618.0, 1550.0, 1519.0, 1519.0, 1742.0, 1892.0, 1966.0, 1966.0, 2092.0, 2040.0, 2048.0, 1997.0, 1953.0, 1946.0, 1926.0, 1791.0, 1497.0, 1112.0, 678.0, 279.0, 0])]\n",
      " [list([464.0, 587.0, 824.0, 1016.0, 1156.0, 1285.0, 1454.0, 1603.0, 1728.0, 1851.0, 1921.0, 1963.0, 1952.0, 1872.0, 1742.0, 1631.0, 1641.0, 1629.0, 1628.0, 1668.0, 1696.0, 1765.0, 1803.0, 1879.0, 1879.0, 1859.0, 1812.0, 1738.0, 1683.0, 1678.0, 1698.0, 1722.0, 1741.0, 1783.0, 1769.0, 1758.0, 1752.0, 1750.0, 1710.0, 1668.0, 1673.0, 1694.0, 1705.0, 1687.0, 1594.0, 1447.0, 1398.0, 1303.0, 1220.0, 1172.0, 1183.0, 1203.0, 1280.0, 1364.0, 1420.0, 1433.0, 1409.0, 1386.0, 1490.0, 1658.0, 1725.0, 1807.0, 1869.0, 1904.0, 1929.0, 1918.0, 1888.0, 1847.0, 1777.0, 1543.0, 1232.0, 812.0, 356.0, 0])]\n",
      " [list([403.0, 534.0, 734.0, 931.0, 1133.0, 1235.0, 1317.0, 1391.0, 1434.0, 1499.0, 1438.0, 1423.0, 1396.0, 1407.0, 1445.0, 1412.0, 1443.0, 1461.0, 1475.0, 1504.0, 1535.0, 1469.0, 1255.0, 1044.0, 794.0, 554.0, 236.0, 0])]\n",
      " ...\n",
      " [list([1416.0, 1265.0, 1542.0, 1964.0, 2060.0, 2060.0, 2010.0, 2035.0, 1993.0, 1935.0, 1935.0, 1949.0, 1844.0, 1651.0, 1571.0, 1554.0, 1574.0, 1496.0, 1357.0, 1045.0, 701.0, 348.0, 94.0, 7.0, 10.0, 12.0, 14.0, 11.0, 29.0, 105.0, 783.0, 970.0, 1294.0, 1389.0, 1514.0, 1479.0, 1439.0, 1055.0, 557.0, 43.0, 11.0, 12.0, 16.0, 17.0, 17.0, 19.0, 19.0, 19.0, 16.0, 16.0, 16.0, 17.0, 16.0, 15.0, 15.0, 13.0, 14.0, 14.0, 17.0, 18.0, 16.0, 17.0, 16.0, 16.0, 13.0, 17.0, 17.0, 16.0, 14.0, 17.0, 15.0, 13.0, 15.0, 17.0, 15.0, 14.0, 13.0, 16.0, 14.0, 15.0, 15.0, 14.0, 16.0, 15.0, 51.0, 665.0, 615.0, 779.0, 939.0, 975.0, 1042.0, 1129.0, 1195.0, 1271.0, 1298.0, 1314.0, 1289.0, 1238.0, 1076.0, 865.0, 562.0, 209.0, 0])]\n",
      " [list([847.0, 707.0, 927.0, 1026.0, 1148.0, 1299.0, 1391.0, 1462.0, 1512.0, 1549.0, 1568.0, 1610.0, 1607.0, 1807.0, 2033.0, 2194.0, 2310.0, 2420.0, 2606.0, 2588.0, 2585.0, 2581.0, 2587.0, 2581.0, 2562.0, 2503.0, 2290.0, 1634.0, 1058.0, 607.0, 0])]\n",
      " [list([656.0, 729.0, 1036.0, 1253.0, 1465.0, 1662.0, 1707.0, 1730.0, 1897.0, 1890.0, 1885.0, 1856.0, 1833.0, 1789.0, 1746.0, 1752.0, 1766.0, 1799.0, 1780.0, 1759.0, 1743.0, 1732.0, 1735.0, 1824.0, 1788.0, 1737.0, 1637.0, 1547.0, 1443.0, 1199.0, 862.0, 400.0, 0])]]\n"
     ]
    }
   ],
   "source": [
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d06cbda8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}