{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Pandas operations for data cleaning and tidying\n",
        "\n",
        "Author: Dr. Hickman\n",
        "\n",
        "### Import"
      ],
      "id": "97e3f886-483a-4b20-984a-1c9bfaa4d939"
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd"
      ],
      "id": "cell-3"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Creation"
      ],
      "id": "fd590c9a-20d8-4569-90be-a7e7e858d169"
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----------------------\n",
            "0    1.0\n",
            "1    3.0\n",
            "2    5.0\n",
            "3    NaN\n",
            "4    6.0\n",
            "5    8.0\n",
            "dtype: float64"
          ]
        }
      ],
      "source": [
        "#SERIES\n",
        "print(\"----------------------\")\n",
        "s = pd.Series([1, 3, 5, np.nan, 6, 8])\n",
        "print(s)\n"
      ],
      "id": "cell-5"
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----------------------\n",
            "     A          B    C  D      E    F\n",
            "0  1.0 2013-01-02  1.0  3   test  foo\n",
            "1  1.0 2013-01-02  1.0  3  train  foo\n",
            "2  1.0 2013-01-02  1.0  3   test  foo\n",
            "3  1.0 2013-01-02  1.0  3  train  foo\n",
            "\n",
            " column types:\n",
            "A           float64\n",
            "B    datetime64[ns]\n",
            "C           float32\n",
            "D             int32\n",
            "E          category\n",
            "F            object\n",
            "dtype: object"
          ]
        }
      ],
      "source": [
        "#DATA-FRAME (FROM DICTIONARY)\n",
        "print(\"----------------------\")\n",
        "df2 = pd.DataFrame(\n",
        "    {\n",
        "        \"A\": 1.0,\n",
        "        \"B\": pd.Timestamp(\"20130102\"),\n",
        "        \"C\": pd.Series(1, index=list(range(4)), dtype=\"float32\"),\n",
        "        \"D\": np.array([3] * 4, dtype=\"int32\"),\n",
        "        \"E\": pd.Categorical([\"test\", \"train\", \"test\", \"train\"]),\n",
        "        \"F\": \"foo\",\n",
        "    }\n",
        ")\n",
        "print(df2)\n",
        "print(\"\\n column types:\")\n",
        "print(df2.dtypes)\n",
        "\n",
        "#"
      ],
      "id": "cell-6"
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "     A     B     C     D\n",
            "0 -2.0   0.0   1.0   2.0\n",
            "1  1.0   2.0   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0"
          ]
        }
      ],
      "source": [
        "#DATA-FRAME (FROM NP ARRAY)\n",
        "values=np.array([[-2,0,1,2],[1,2,3,4],[5,6,7.,8],[9,10,11,12]])\n",
        "df = pd.DataFrame(values, columns=list(\"ABCD\"))\n",
        "print(df)"
      ],
      "id": "cell-7"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Pandas basics"
      ],
      "id": "bc24071d-c9a3-4530-b1b6-f45637f009fa"
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [],
      "source": [
        "df = pd.read_csv('data/mpg.csv')"
      ],
      "id": "cell-9"
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----------------------\n",
            "GENERAL:\n",
            "----------------------\n",
            "RangeIndex(start=0, stop=234, step=1)\n",
            "Index(['Unnamed: 0', 'manufacturer', 'model', 'displ', 'year', 'cyl', 'trans',\n",
            "       'drv', 'cty', 'hwy', 'fl', 'class'],\n",
            "      dtype='object')\n",
            "number of rows: 234\n",
            "number of col: 12\n",
            "keys: Index(['Unnamed: 0', 'manufacturer', 'model', 'displ', 'year', 'cyl', 'trans',\n",
            "       'drv', 'cty', 'hwy', 'fl', 'class'],\n",
            "      dtype='object') <class 'pandas.core.indexes.base.Index'>"
          ]
        }
      ],
      "source": [
        "print(\"----------------------\")\n",
        "print(\"GENERAL:\")\n",
        "print(\"----------------------\")\n",
        "print(df.index)\n",
        "print(df.columns)\n",
        "print(\"number of rows:\", len(df.index))\n",
        "print(\"number of col:\",  len(df.columns))\n",
        "print(\"keys:\",  df.keys(),type(df.keys()))\n"
      ],
      "id": "cell-10"
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "RangeIndex: 234 entries, 0 to 233\n",
            "Data columns (total 12 columns):\n",
            " #   Column        Non-Null Count  Dtype  \n",
            "---  ------        --------------  -----  \n",
            " 0   Unnamed: 0    234 non-null    int64  \n",
            " 1   manufacturer  234 non-null    object \n",
            " 2   model         234 non-null    object \n",
            " 3   displ         234 non-null    float64\n",
            " 4   year          234 non-null    int64  \n",
            " 5   cyl           234 non-null    int64  \n",
            " 6   trans         234 non-null    object \n",
            " 7   drv           234 non-null    object \n",
            " 8   cty           234 non-null    int64  \n",
            " 9   hwy           234 non-null    int64  \n",
            " 10  fl            234 non-null    object \n",
            " 11  class         234 non-null    object \n",
            "dtypes: float64(1), int64(5), object(6)\n",
            "memory usage: 22.1+ KB\n",
            "info None\n",
            "head    Unnamed: 0 manufacturer model  displ  year  cyl       trans drv  cty  hwy  \\\n",
            "0           1         audi    a4    1.8  1999    4    auto(l5)   f   18   29   \n",
            "1           2         audi    a4    1.8  1999    4  manual(m5)   f   21   29   \n",
            "2           3         audi    a4    2.0  2008    4  manual(m6)   f   20   31   \n",
            "3           4         audi    a4    2.0  2008    4    auto(av)   f   21   30   \n",
            "4           5         audi    a4    2.8  1999    6    auto(l5)   f   16   26   \n",
            "\n",
            "  fl    class  \n",
            "0  p  compact  \n",
            "1  p  compact  \n",
            "2  p  compact  \n",
            "3  p  compact  \n",
            "4  p  compact  \n",
            "TYPES Unnamed: 0        int64\n",
            "manufacturer     object\n",
            "model            object\n",
            "displ           float64\n",
            "year              int64\n",
            "cyl               int64\n",
            "trans            object\n",
            "drv              object\n",
            "cty               int64\n",
            "hwy               int64\n",
            "fl               object\n",
            "class            object\n",
            "dtype: object"
          ]
        }
      ],
      "source": [
        "print(\"info\",df.info() )\n",
        "print(\"head\", df.head())\n",
        "print(\"TYPES\", df.dtypes)"
      ],
      "id": "cell-11"
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "       Unnamed: 0       displ         year         cyl         cty         hwy\n",
            "count  234.000000  234.000000   234.000000  234.000000  234.000000  234.000000\n",
            "mean   117.500000    3.471795  2003.500000    5.888889   16.858974   23.440171\n",
            "std     67.694165    1.291959     4.509646    1.611534    4.255946    5.954643\n",
            "min      1.000000    1.600000  1999.000000    4.000000    9.000000   12.000000\n",
            "25%     59.250000    2.400000  1999.000000    4.000000   14.000000   18.000000\n",
            "50%    117.500000    3.300000  2003.500000    6.000000   17.000000   24.000000\n",
            "75%    175.750000    4.600000  2008.000000    8.000000   19.000000   27.000000\n",
            "max    234.000000    7.000000  2008.000000    8.000000   35.000000   44.000000"
          ]
        }
      ],
      "source": [
        "\n",
        "# (\"----------------------\")\n",
        "# (\"BASIC STATISTICS:\")\n",
        "# (\"----------------------\")\n",
        "print(df.describe())"
      ],
      "id": "cell-12"
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "            Unnamed: 0     displ      year       cyl       cty       hwy\n",
            "Unnamed: 0    1.000000 -0.402459 -0.073577 -0.407698  0.401822  0.327046\n",
            "displ        -0.402459  1.000000  0.147843  0.930227 -0.798524 -0.766020\n",
            "year         -0.073577  0.147843  1.000000  0.122245 -0.037232  0.002158\n",
            "cyl          -0.407698  0.930227  0.122245  1.000000 -0.805771 -0.761912\n",
            "cty           0.401822 -0.798524 -0.037232 -0.805771  1.000000  0.955916\n",
            "hwy           0.327046 -0.766020  0.002158 -0.761912  0.955916  1.000000"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/var/folders/0l/7994y9sn0l1d6cxc7lqc26p80000gn/T/ipykernel_18079/2731510891.py:4: FutureWarning: The default value of numeric_only in DataFrame.corr is deprecated. In a future version, it will default to False. Select only valid columns or specify the value of numeric_only to silence this warning.\n",
            "  print(df.corr())"
          ]
        }
      ],
      "source": [
        "\n",
        "# (\"----------------------\")\n",
        "# (\"CORRELATION MATRIX:\")\n",
        "# (\"----------------------\")\n",
        "print(df.corr())"
      ],
      "id": "cell-13"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Example"
      ],
      "id": "9255d367-cbcf-410b-8f89-d1ebb1c38026"
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "     A     B     C     D\n",
            "0 -2.0   0.0   1.0   2.0\n",
            "1  1.0   2.0   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0"
          ]
        }
      ],
      "source": [
        "#DATA-FRAME (FROM NP ARRAY)\n",
        "# np.random.seed(seed=123243)\n",
        "values=np.array([[-2,0,1,2],[1,2,3,4],[5,6,7.,8],[9,10,11,12]])\n",
        "df = pd.DataFrame(values, columns=list(\"ABCD\"))\n",
        "print(df)"
      ],
      "id": "cell-15"
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "------A------\n",
            "      A     B     C     D\n",
            "0 -2.0   0.0   1.0   2.0\n",
            "1  1.0   2.0   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0\n",
            "------B------\n",
            " A    float64\n",
            "B    float64\n",
            "C    float64\n",
            "D    float64\n",
            "dtype: object\n",
            "     A     B     C     D\n",
            "0 -2.0   0.0   1.0   2.0\n",
            "1  1.0   2.0   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0\n",
            "     A     B     C     D\n",
            "1  1.0   2.0   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0\n",
            "RangeIndex(start=0, stop=4, step=1)\n",
            "Index(['A', 'B', 'C', 'D'], dtype='object')\n",
            "     0    1    2     3\n",
            "A -2.0  1.0  5.0   9.0\n",
            "B  0.0  2.0  6.0  10.0\n",
            "C  1.0  3.0  7.0  11.0\n",
            "D  2.0  4.0  8.0  12.0\n",
            "------C------\n",
            " [[-2.  0.  1.  2.]\n",
            " [ 1.  2.  3.  4.]\n",
            " [ 5.  6.  7.  8.]\n",
            " [ 9. 10. 11. 12.]]\n",
            "------D------\n",
            "               A          B          C          D\n",
            "count  4.000000   4.000000   4.000000   4.000000\n",
            "mean   3.250000   4.500000   5.500000   6.500000\n",
            "std    4.787136   4.434712   4.434712   4.434712\n",
            "min   -2.000000   0.000000   1.000000   2.000000\n",
            "25%    0.250000   1.500000   2.500000   3.500000\n",
            "50%    3.000000   4.000000   5.000000   6.000000\n",
            "75%    6.000000   7.000000   8.000000   9.000000\n",
            "max    9.000000  10.000000  11.000000  12.000000\n",
            "------E------\n",
            "      A     B     C     D\n",
            "3  9.0  10.0  11.0  12.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "1  1.0   2.0   3.0   4.0\n",
            "0 -2.0   0.0   1.0   2.0\n",
            "------F------\n",
            "      A     B     C     D\n",
            "0 -2.0   0.0   1.0   2.0\n",
            "1  1.0   2.0   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0"
          ]
        }
      ],
      "source": [
        "\n",
        "#VIEWING DATA\n",
        "print('------A------\\n',df)\n",
        "print('------B------\\n',df.dtypes)\n",
        "print(df.head())\n",
        "print(df.tail(3))\n",
        "print(df.index)\n",
        "print(df.columns)\n",
        "print(df.T)                                                         #TRANSPOSE\n",
        "print('------C------\\n',df.to_numpy())\n",
        "print('------D------\\n',df.describe())                              #GET BASIC STATISTICS \n",
        "print('------E------\\n',df.sort_index(axis=0, ascending=False))     #SORT BY AXIS\n",
        "print('------F------\\n',df.sort_values(by=\"B\"))                     #SORT BY VALUES \n"
      ],
      "id": "cell-16"
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "------G------\n",
            " 0   -2.0\n",
            "1    1.0\n",
            "2    5.0\n",
            "3    9.0\n",
            "Name: A, dtype: float64\n",
            "------H------\n",
            "      A    B    C    D\n",
            "0 -2.0  0.0  1.0  2.0\n",
            "1  1.0  2.0  3.0  4.0\n",
            "2  5.0  6.0  7.0  8.0\n",
            "------I------\n",
            " 0   -2.0\n",
            "1    1.0\n",
            "2    5.0\n",
            "Name: A, dtype: float64\n",
            "------K------\n",
            "      A     B\n",
            "0 -2.0   0.0\n",
            "1  1.0   2.0\n",
            "2  5.0   6.0\n",
            "3  9.0  10.0"
          ]
        }
      ],
      "source": [
        "#SELECTION\n",
        "print('------G------\\n',df[\"A\"])                                    #GET COLUMN A\n",
        "print('------H------\\n',df[0:3])                                    #SLICE BY ROWS\n",
        "print('------I------\\n',df[0:3][\"A\"])\n",
        "print('------K------\\n',df.loc[:, [\"A\", \"B\"]])                      \n"
      ],
      "id": "cell-17"
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "------O------\n",
            " A     9.0\n",
            "B    10.0\n",
            "C    11.0\n",
            "D    12.0\n",
            "Name: 3, dtype: float64\n",
            "------P------\n",
            "      A    B\n",
            "1  1.0  2.0\n",
            "------Q------\n",
            "      A     C\n",
            "1  1.0   3.0\n",
            "2  5.0   7.0\n",
            "3  9.0  11.0\n",
            "------R------\n",
            "      A    B    C    D\n",
            "1  1.0  2.0  3.0  4.0\n",
            "2  5.0  6.0  7.0  8.0\n",
            "------S------\n",
            " 2.0"
          ]
        }
      ],
      "source": [
        "#SELECT BY INDEX VALUE \"LOCATION\" (SIMILAR TO NUMPY)\n",
        "print('------O------\\n',df.iloc[3])                              \n",
        "print('------P------\\n',df.iloc[1:2, 0:2])                               \n",
        "print('------Q------\\n',df.iloc[[1, 2, 3], [0, 2]])\n",
        "print('------R------\\n',df.iloc[1:3, :])\n",
        "print('------S------\\n',df.iloc[1, 1])"
      ],
      "id": "cell-18"
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "------BOOLEAN INDEXING------\n",
            "     A     B     C     D\n",
            "1  1.0   2.0   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0\n",
            "------------\n",
            "      A     B     C     D\n",
            "0  NaN   NaN   1.0   2.0\n",
            "1  1.0   2.0   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0\n",
            "------------\n",
            "      A     B     C     D      E\n",
            "0 -2.0   0.0   1.0   2.0    one\n",
            "1  1.0   2.0   3.0   4.0    two\n",
            "2  5.0   6.0   7.0   8.0  three\n",
            "3  9.0  10.0  11.0  12.0   four\n",
            "------------\n",
            "      A     B     C     D     E\n",
            "1  1.0   2.0   3.0   4.0   two\n",
            "3  9.0  10.0  11.0  12.0  four\n"
          ]
        }
      ],
      "source": [
        "print(\"------BOOLEAN INDEXING------\")\n",
        "print(df[df[\"A\"] > 0])\n",
        "print('------------\\n',df[df > 0])\n",
        "df2 = df.copy()\n",
        "df2[\"E\"] = [\"one\", \"two\", \"three\", \"four\"]\n",
        "print('------------\\n',df2)\n",
        "print('------------\\n',df2[df2[\"E\"].isin([\"two\", \"four\"])])         #SELECT VARIABLE IN LIST\n",
        "\n",
        "# BOOLEAN EXAMPLES\n",
        "print()\n",
        "#       (Expression 1)  AND   (Expression 2)\n",
        "# print(   (Sales > 15)    &   (Newspaper > 10)  )\n"
      ],
      "id": "cell-19"
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "     A     B     C     D\n",
            "0 -2.0   0.0   1.0   2.0\n",
            "1  1.0   2.0   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0\n",
            "       A      B     C     D\n",
            "0  False  False  True  True\n",
            "1   True   True  True  True\n",
            "2   True   True  True  True\n",
            "3   True   True  True  True"
          ]
        }
      ],
      "source": [
        "print(df)\n",
        "print(df > 0)"
      ],
      "id": "cell-20"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Convert to datetime objects"
      ],
      "id": "59dcdcd0-b470-4f5c-8c82-0e41bc4ef84c"
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BEFORE\n",
            "    0       1       2       3         4\n",
            "0  1  210298  022198  980221  19980221\n",
            "1  2  190399  031999  990319  19990319\n",
            "AFTER\n",
            "    0       1          2          3          4\n",
            "0  1  210298 1998-02-21 1998-02-21 1998-02-21\n",
            "1  2  190399 1999-03-19 1999-03-19 1999-03-19"
          ]
        }
      ],
      "source": [
        "# CREATE DATA \n",
        "df = pd.DataFrame(\n",
        "    [\n",
        "        [1, \"210298\", \"022198\", \"980221\", \"19980221\"],\n",
        "        [2, \"190399\", \"031999\", \"990319\", \"19990319\"],\n",
        "    ]\n",
        ")\n",
        "\n",
        "print(\"BEFORE\\n\",df)\n",
        "df[4] = pd.to_datetime(df[4])  #convert column-4\n",
        "df[3] = pd.to_datetime(df[3])  #convert column-3\n",
        "df[2] = pd.to_datetime(df[2])  #convert column-2\n",
        "print(\"AFTER\\n\",df)\n"
      ],
      "id": "cell-22"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Renaming, randomizing, reductions"
      ],
      "id": "7b91d1ac-9e2a-45fc-851b-797d0b7aaa51"
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ORIGINAL DATA-FRAME\n",
            "       0  1   2\n",
            "0   pos  M   1\n",
            "1   pos  M   2\n",
            "2   neg  M   3\n",
            "3   neg  M   4\n",
            "4   sev  M   6\n",
            "5   sev  M   7\n",
            "6   pos  F   8\n",
            "7   pos  F   9\n",
            "8   neg  F  10\n",
            "9   neg  F  11\n",
            "10  sev  F  12\n",
            "11  sev  F  13"
          ]
        }
      ],
      "source": [
        "# AVERAGING\n",
        "df = pd.DataFrame(\n",
        "    [\n",
        "        [\"pos\", \"M\", 1],\n",
        "        [\"pos\", \"M\", 2],\n",
        "        [\"neg\", \"M\", 3],\n",
        "        [\"neg\", \"M\", 4],\n",
        "        [\"sev\", \"M\", 6],\n",
        "        [\"sev\", \"M\", 7],\n",
        "        [\"pos\", \"F\", 8],\n",
        "        [\"pos\", \"F\", 9],\n",
        "        [\"neg\", \"F\", 10],\n",
        "        [\"neg\", \"F\", 11],\n",
        "        [\"sev\", \"F\", 12],\n",
        "        [\"sev\", \"F\", 13],\n",
        "    ]\n",
        ")\n",
        "print(\"ORIGINAL DATA-FRAME\\n\",df)"
      ],
      "id": "cell-24"
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   thrombosis m_f  num\n",
            "0         pos   M    1\n",
            "1         pos   M    2\n",
            "2         neg   M    3\n",
            "3         neg   M    4\n",
            "4         sev   M    6\n",
            "5         sev   M    7\n",
            "6         pos   F    8\n",
            "7         pos   F    9\n",
            "8         neg   F   10\n",
            "9         neg   F   11\n",
            "10        sev   F   12\n",
            "11        sev   F   13"
          ]
        }
      ],
      "source": [
        "# RENAMING COLUMNS \n",
        "df=df.rename(columns={0: \"thrombosis\", 1: \"m_f\", 2: \"num\"})\n",
        "print(df)"
      ],
      "id": "cell-25"
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "   thrombosis m_f  num\n",
            "0         sev   F   13\n",
            "1         pos   M    1\n",
            "2         sev   M    7\n",
            "3         neg   F   11\n",
            "4         sev   F   12\n",
            "5         neg   M    4\n",
            "6         pos   F    8\n",
            "7         neg   M    3\n",
            "8         sev   M    6\n",
            "9         pos   M    2\n",
            "10        neg   F   10\n",
            "11        pos   F    9"
          ]
        }
      ],
      "source": [
        "# RANDOMIZE ROWS AND RESET INDEX\n",
        "df=df.sample(frac=1).reset_index(drop=True)\n",
        "print(df)"
      ],
      "id": "cell-26"
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  m_f  num\n",
            "0   F   63\n",
            "1   M   23"
          ]
        }
      ],
      "source": [
        "# SUM THE \"num\" BASED ON \"m_f\" CATEGORY\n",
        "print(df.groupby(['m_f'])['num'].sum().reset_index())"
      ],
      "id": "cell-27"
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  m_f thrombosis  num\n",
            "0   F        neg   21\n",
            "1   F        pos   17\n",
            "2   F        sev   25\n",
            "3   M        neg    7\n",
            "4   M        pos    3\n",
            "5   M        sev   13"
          ]
        }
      ],
      "source": [
        "# SUM THE \"num\" BASED ON \"m_f\" AND \"thrombosis\" CATEGORY\n",
        "print(df.groupby(['m_f','thrombosis'])['num'].sum().reset_index())"
      ],
      "id": "cell-28"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### GAP-MINDER EXAMPLE"
      ],
      "id": "550aeec2-091e-4480-86fe-5f93005bb8c8"
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {},
      "outputs": [],
      "source": [
        "# read data\n",
        "df = pd.read_csv('./data/gapminder.csv')\n"
      ],
      "id": "cell-30"
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "       country continent  year  lifeExp       pop   gdpPercap\n",
            "0  Afghanistan      Asia  1952   28.801   8425333  779.445314\n",
            "1  Afghanistan      Asia  1957   30.332   9240934  820.853030\n",
            "2  Afghanistan      Asia  1962   31.997  10267083  853.100710\n",
            "3  Afghanistan      Asia  1967   34.020  11537966  836.197138\n",
            "4  Afghanistan      Asia  1972   36.088  13079460  739.981106\n",
            "5  Afghanistan      Asia  1977   38.438  14880372  786.113360\n",
            "6  Afghanistan      Asia  1982   39.854  12881816  978.011439\n",
            "7  Afghanistan      Asia  1987   40.822  13867957  852.395945\n",
            "8  Afghanistan      Asia  1992   41.674  16317921  649.341395\n",
            "9  Afghanistan      Asia  1997   41.763  22227415  635.341351"
          ]
        }
      ],
      "source": [
        "# print original df\n",
        "print(df.head(10))"
      ],
      "id": "cell-31"
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(1704, 6)\n",
            "<class 'pandas.core.frame.DataFrame'>"
          ]
        }
      ],
      "source": [
        "# print df size\n",
        "print(df.shape)\n",
        "\n",
        "print(type(df))"
      ],
      "id": "cell-32"
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {},
      "outputs": [],
      "source": [
        "# GROUP BY CONTINENT \n",
        "# print((df.groupby('continent')).head())\n",
        "# print('----\\n',df.groupby('continent').tail())\n"
      ],
      "id": "cell-33"
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----AVERAGE LIFE EXPECTANCY BY CONTINENT----\n",
            "continent\n",
            "Africa      48.865330\n",
            "Americas    64.658737\n",
            "Asia        60.064903\n",
            "Europe      71.903686\n",
            "Oceania     74.326208\n",
            "Name: lifeExp, dtype: float64"
          ]
        }
      ],
      "source": [
        "print(\"----AVERAGE LIFE EXPECTANCY BY CONTINENT----\")\n",
        "print(df.groupby('continent')['lifeExp'].mean())"
      ],
      "id": "cell-34"
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----AVERAGE LIFE EXPECTANCY BY COUNTRY----\n",
            "country\n",
            "Afghanistan           37.478833\n",
            "Albania               68.432917\n",
            "Algeria               59.030167\n",
            "Angola                37.883500\n",
            "Argentina             69.060417\n",
            "                        ...    \n",
            "Vietnam               57.479500\n",
            "West Bank and Gaza    60.328667\n",
            "Yemen, Rep.           46.780417\n",
            "Zambia                45.996333\n",
            "Zimbabwe              52.663167\n",
            "Name: lifeExp, Length: 142, dtype: float64"
          ]
        }
      ],
      "source": [
        "\n",
        "print(\"----AVERAGE LIFE EXPECTANCY BY COUNTRY----\")\n",
        "print(df.groupby('country')['lifeExp'].mean())\n"
      ],
      "id": "cell-35"
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----AVERAGE LIFE EXPECTANCY BY CONTINENT AND YEAR----\n",
            "continent  year\n",
            "Africa     1952    39.135500\n",
            "           1957    41.266346\n",
            "           1962    43.319442\n",
            "           1967    45.334538\n",
            "           1972    47.450942\n",
            "           1977    49.580423\n",
            "           1982    51.592865\n",
            "           1987    53.344788\n",
            "           1992    53.629577\n",
            "           1997    53.598269\n",
            "           2002    53.325231\n",
            "           2007    54.806038\n",
            "Americas   1952    53.279840\n",
            "           1957    55.960280\n",
            "           1962    58.398760\n",
            "           1967    60.410920\n",
            "           1972    62.394920\n",
            "           1977    64.391560\n",
            "           1982    66.228840\n",
            "           1987    68.090720\n",
            "           1992    69.568360\n",
            "           1997    71.150480\n",
            "           2002    72.422040\n",
            "           2007    73.608120\n",
            "Asia       1952    46.314394\n",
            "           1957    49.318544\n",
            "           1962    51.563223\n",
            "           1967    54.663640\n",
            "           1972    57.319269\n",
            "           1977    59.610556\n",
            "           1982    62.617939\n",
            "           1987    64.851182\n",
            "           1992    66.537212\n",
            "           1997    68.020515\n",
            "           2002    69.233879\n",
            "           2007    70.728485\n",
            "Europe     1952    64.408500\n",
            "           1957    66.703067\n",
            "           1962    68.539233\n",
            "           1967    69.737600\n",
            "           1972    70.775033\n",
            "           1977    71.937767\n",
            "           1982    72.806400\n",
            "           1987    73.642167\n",
            "           1992    74.440100\n",
            "           1997    75.505167\n",
            "           2002    76.700600\n",
            "           2007    77.648600\n",
            "Oceania    1952    69.255000\n",
            "           1957    70.295000\n",
            "           1962    71.085000\n",
            "           1967    71.310000\n",
            "           1972    71.910000\n",
            "           1977    72.855000\n",
            "           1982    74.290000\n",
            "           1987    75.320000\n",
            "           1992    76.945000\n",
            "           1997    78.190000\n",
            "           2002    79.740000\n",
            "           2007    80.719500\n",
            "Name: lifeExp, dtype: float64"
          ]
        }
      ],
      "source": [
        "\n",
        "print(\"----AVERAGE LIFE EXPECTANCY BY CONTINENT AND YEAR----\")\n",
        "print(df.groupby(['continent','year'])['lifeExp'].mean())\n",
        "\n"
      ],
      "id": "cell-36"
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----AVERAGE+MEDIAN EXPECTANCY BY YEAR----\n",
            "           mean   median\n",
            "year                    \n",
            "1952  49.057620  45.1355\n",
            "1957  51.507401  48.3605\n",
            "1962  53.609249  50.8810\n",
            "1967  55.678290  53.8250\n",
            "1972  57.647386  56.5300\n",
            "1977  59.570157  59.6720\n",
            "1982  61.533197  62.4415\n",
            "1987  63.212613  65.8340\n",
            "1992  64.160338  67.7030\n",
            "1997  65.014676  69.3940\n",
            "2002  65.694923  70.8255\n",
            "2007  67.007423  71.9355"
          ]
        }
      ],
      "source": [
        "print(\"----AVERAGE+MEDIAN EXPECTANCY BY YEAR----\")\n",
        "print(df.groupby('year')['lifeExp'].agg([np.mean,np.median]))"
      ],
      "id": "cell-37"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Mutate,select,split,filter,arrange"
      ],
      "id": "f1843e62-d9a2-4fb2-93a5-5d3f1c0e2489"
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----ORIGINAL----\n",
            "   Unnamed: 0 manufacturer       model  displ  year  cyl       trans drv  cty  \\\n",
            "0           1         audi          a4    1.8  1999    4    auto(l5)   f   18   \n",
            "1           2         audi          a4    1.8  1999    4  manual(m5)   f   21   \n",
            "2           3         audi          a4    2.0  2008    4  manual(m6)   f   20   \n",
            "3           4         audi          a4    2.0  2008    4    auto(av)   f   21   \n",
            "4           5         audi          a4    2.8  1999    6    auto(l5)   f   16   \n",
            "5           6         audi          a4    2.8  1999    6  manual(m5)   f   18   \n",
            "6           7         audi          a4    3.1  2008    6    auto(av)   f   18   \n",
            "7           8         audi  a4 quattro    1.8  1999    4  manual(m5)   4   18   \n",
            "8           9         audi  a4 quattro    1.8  1999    4    auto(l5)   4   16   \n",
            "9          10         audi  a4 quattro    2.0  2008    4  manual(m6)   4   20   \n",
            "\n",
            "   hwy fl    class  \n",
            "0   29  p  compact  \n",
            "1   29  p  compact  \n",
            "2   31  p  compact  \n",
            "3   30  p  compact  \n",
            "4   26  p  compact  \n",
            "5   26  p  compact  \n",
            "6   27  p  compact  \n",
            "7   26  p  compact  \n",
            "8   25  p  compact  \n",
            "9   28  p  compact  "
          ]
        }
      ],
      "source": [
        "print(\"----ORIGINAL----\")\n",
        "mpg = pd.read_csv('data/mpg.csv')\n",
        "print(mpg.head(10))\n"
      ],
      "id": "cell-39"
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----MUTATE----\n",
            "   Unnamed: 0 manufacturer model  displ  year  cyl       trans drv       cty  \\\n",
            "0           1         audi    a4    1.8  1999    4    auto(l5)   f  7.650975   \n",
            "1           2         audi    a4    1.8  1999    4  manual(m5)   f  8.926137   \n",
            "2           3         audi    a4    2.0  2008    4  manual(m6)   f  8.501083   \n",
            "3           4         audi    a4    2.0  2008    4    auto(av)   f  8.926137   \n",
            "4           5         audi    a4    2.8  1999    6    auto(l5)   f  6.800866   \n",
            "\n",
            "         hwy fl    class        avg  \n",
            "0  12.326571  p  compact   9.988773  \n",
            "1  12.326571  p  compact  10.626354  \n",
            "2  13.176679  p  compact  10.838881  \n",
            "3  12.751625  p  compact  10.838881  \n",
            "4  11.051408  p  compact   8.926137  "
          ]
        }
      ],
      "source": [
        "print(\"----MUTATE----\")\n",
        "mpg['cty'] = mpg['cty'] * 1.609/3.7854 # mile/gallon --> km/l\n",
        "mpg['hwy'] = mpg['hwy'] * 1.609/3.7854\n",
        "mpg['avg'] = (mpg.cty + mpg.hwy)/2\n",
        "print(mpg.head())"
      ],
      "id": "cell-40"
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----SPLIT----\n",
            "     Unnamed: 0 manufacturer   model  displ  year  cyl       trans drv  \\\n",
            "0             1         audi      a4    1.8  1999    4    auto(l5)   f   \n",
            "1             2         audi      a4    1.8  1999    4  manual(m5)   f   \n",
            "2             3         audi      a4    2.0  2008    4  manual(m6)   f   \n",
            "3             4         audi      a4    2.0  2008    4    auto(av)   f   \n",
            "4             5         audi      a4    2.8  1999    6    auto(l5)   f   \n",
            "..          ...          ...     ...    ...   ...  ...         ...  ..   \n",
            "229         230   volkswagen  passat    2.0  2008    4    auto(s6)   f   \n",
            "230         231   volkswagen  passat    2.0  2008    4  manual(m6)   f   \n",
            "231         232   volkswagen  passat    2.8  1999    6    auto(l5)   f   \n",
            "232         233   volkswagen  passat    2.8  1999    6  manual(m5)   f   \n",
            "233         234   volkswagen  passat    3.6  2008    6    auto(s6)   f   \n",
            "\n",
            "          cty        hwy fl    class        avg  0  1  \n",
            "0    7.650975  12.326571  p  compact   9.988773  1  8  \n",
            "1    8.926137  12.326571  p  compact  10.626354  1  8  \n",
            "2    8.501083  13.176679  p  compact  10.838881  2  0  \n",
            "3    8.926137  12.751625  p  compact  10.838881  2  0  \n",
            "4    6.800866  11.051408  p  compact   8.926137  2  8  \n",
            "..        ...        ... ..      ...        ... .. ..  \n",
            "229  8.076029  11.901516  p  midsize   9.988773  2  0  \n",
            "230  8.926137  12.326571  p  midsize  10.626354  2  0  \n",
            "231  6.800866  11.051408  p  midsize   8.926137  2  8  \n",
            "232  7.650975  11.051408  p  midsize   9.351191  2  8  \n",
            "233  7.225921  11.051408  p  midsize   9.138664  3  6  \n",
            "\n",
            "[234 rows x 15 columns]"
          ]
        }
      ],
      "source": [
        "#SPLIT STRING COLUMN by delimiter \".\" AND ADD THE TWO PARTS AS NEW COLUMNS\n",
        "# just prints, doesn't overwrite original\n",
        "print(\"----SPLIT----\")\n",
        "print(pd.concat([mpg, mpg['displ'].astype(\"str\").str.split('.', expand=True)], axis=1))"
      ],
      "id": "cell-41"
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----SELECT----\n",
            "     year manufacturer        avg\n",
            "0    1999         audi   9.988773\n",
            "1    1999         audi  10.626354\n",
            "2    2008         audi  10.838881\n",
            "3    2008         audi  10.838881\n",
            "4    1999         audi   8.926137\n",
            "..    ...          ...        ...\n",
            "229  2008   volkswagen   9.988773\n",
            "230  2008   volkswagen  10.626354\n",
            "231  1999   volkswagen   8.926137\n",
            "232  1999   volkswagen   9.351191\n",
            "233  2008   volkswagen   9.138664\n",
            "\n",
            "[234 rows x 3 columns]"
          ]
        }
      ],
      "source": [
        "print(\"----SELECT----\")\n",
        "print(mpg[['year','manufacturer','avg']])\n"
      ],
      "id": "cell-42"
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----FILTER----\n",
            "    Unnamed: 0 manufacturer       model  displ  year  cyl       trans drv  \\\n",
            "0            1         audi          a4    1.8  1999    4    auto(l5)   f   \n",
            "1            2         audi          a4    1.8  1999    4  manual(m5)   f   \n",
            "2            3         audi          a4    2.0  2008    4  manual(m6)   f   \n",
            "3            4         audi          a4    2.0  2008    4    auto(av)   f   \n",
            "4            5         audi          a4    2.8  1999    6    auto(l5)   f   \n",
            "5            6         audi          a4    2.8  1999    6  manual(m5)   f   \n",
            "6            7         audi          a4    3.1  2008    6    auto(av)   f   \n",
            "7            8         audi  a4 quattro    1.8  1999    4  manual(m5)   4   \n",
            "8            9         audi  a4 quattro    1.8  1999    4    auto(l5)   4   \n",
            "9           10         audi  a4 quattro    2.0  2008    4  manual(m6)   4   \n",
            "10          11         audi  a4 quattro    2.0  2008    4    auto(s6)   4   \n",
            "11          12         audi  a4 quattro    2.8  1999    6    auto(l5)   4   \n",
            "12          13         audi  a4 quattro    2.8  1999    6  manual(m5)   4   \n",
            "13          14         audi  a4 quattro    3.1  2008    6    auto(s6)   4   \n",
            "14          15         audi  a4 quattro    3.1  2008    6  manual(m6)   4   \n",
            "15          16         audi  a6 quattro    2.8  1999    6    auto(l5)   4   \n",
            "16          17         audi  a6 quattro    3.1  2008    6    auto(s6)   4   \n",
            "17          18         audi  a6 quattro    4.2  2008    8    auto(s6)   4   \n",
            "\n",
            "         cty        hwy fl    class        avg  \n",
            "0   7.650975  12.326571  p  compact   9.988773  \n",
            "1   8.926137  12.326571  p  compact  10.626354  \n",
            "2   8.501083  13.176679  p  compact  10.838881  \n",
            "3   8.926137  12.751625  p  compact  10.838881  \n",
            "4   6.800866  11.051408  p  compact   8.926137  \n",
            "5   7.650975  11.051408  p  compact   9.351191  \n",
            "6   7.650975  11.476462  p  compact   9.563718  \n",
            "7   7.650975  11.051408  p  compact   9.351191  \n",
            "8   6.800866  10.626354  p  compact   8.713610  \n",
            "9   8.501083  11.901516  p  compact  10.201300  \n",
            "10  8.076029  11.476462  p  compact   9.776246  \n",
            "11  6.375812  10.626354  p  compact   8.501083  \n",
            "12  7.225921  10.626354  p  compact   8.926137  \n",
            "13  7.225921  10.626354  p  compact   8.926137  \n",
            "14  6.375812  10.626354  p  compact   8.501083  \n",
            "15  6.375812  10.201300  p  midsize   8.288556  \n",
            "16  7.225921  10.626354  p  midsize   8.926137  \n",
            "17  6.800866   9.776246  p  midsize   8.288556  "
          ]
        }
      ],
      "source": [
        "print(\"----FILTER----\")  \n",
        "print(mpg[mpg.manufacturer=='audi'])\n"
      ],
      "id": "cell-43"
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----ARRANGE----\n",
            "     Unnamed: 0 manufacturer                model  displ  year  cyl  \\\n",
            "221         222   volkswagen           new beetle    1.9  1999    4   \n",
            "212         213   volkswagen                jetta    1.9  1999    4   \n",
            "222         223   volkswagen           new beetle    1.9  1999    4   \n",
            "196         197       toyota              corolla    1.8  2008    4   \n",
            "195         196       toyota              corolla    1.8  1999    4   \n",
            "..          ...          ...                  ...    ...   ...  ...   \n",
            "126         127         jeep   grand cherokee 4wd    4.7  2008    8   \n",
            "59           60        dodge          durango 4wd    4.7  2008    8   \n",
            "65           66        dodge  ram 1500 pickup 4wd    4.7  2008    8   \n",
            "54           55        dodge    dakota pickup 4wd    4.7  2008    8   \n",
            "69           70        dodge  ram 1500 pickup 4wd    4.7  2008    8   \n",
            "\n",
            "          trans drv        cty        hwy fl       class        avg  \n",
            "221  manual(m5)   f  14.876895  18.702383  d  subcompact  16.789639  \n",
            "212  manual(m5)   f  14.026787  18.702383  d     compact  16.364585  \n",
            "222    auto(l4)   f  12.326571  17.427220  d  subcompact  14.876895  \n",
            "196  manual(m5)   f  11.901516  15.727004  r     compact  13.814260  \n",
            "195  manual(m5)   f  11.051408  14.876895  r     compact  12.964152  \n",
            "..          ...  ..        ...        ... ..         ...        ...  \n",
            "126    auto(l5)   4   3.825487   5.100650  e         suv   4.463069  \n",
            "59     auto(l5)   4   3.825487   5.100650  e         suv   4.463069  \n",
            "65     auto(l5)   4   3.825487   5.100650  e      pickup   4.463069  \n",
            "54     auto(l5)   4   3.825487   5.100650  e      pickup   4.463069  \n",
            "69   manual(m6)   4   3.825487   5.100650  e      pickup   4.463069  \n",
            "\n",
            "[234 rows x 13 columns]"
          ]
        }
      ],
      "source": [
        "\n",
        "#-----------------------------\n",
        "print(\"----ARRANGE----\")  \n",
        "#-----------------------------\n",
        "print(mpg.sort_values(by=['avg'], ascending=False))\n"
      ],
      "id": "cell-44"
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----CONVERT----\n",
            "Unnamed: 0        int64\n",
            "manufacturer     object\n",
            "model            object\n",
            "displ           float64\n",
            "year              int64\n",
            "cyl               int64\n",
            "trans            object\n",
            "drv              object\n",
            "cty             float64\n",
            "hwy             float64\n",
            "fl               object\n",
            "class            object\n",
            "avg             float64\n",
            "dtype: object\n",
            "--------\n",
            "Unnamed: 0         int64\n",
            "manufacturer    category\n",
            "model             string\n",
            "displ            float64\n",
            "year               int64\n",
            "cyl                int64\n",
            "trans             object\n",
            "drv               object\n",
            "cty              float64\n",
            "hwy              float64\n",
            "fl                object\n",
            "class             object\n",
            "avg              float64\n",
            "dtype: object"
          ]
        }
      ],
      "source": [
        "print(\"----CONVERT----\")  \n",
        "print(mpg.dtypes)\n",
        "mpg['manufacturer']=mpg.manufacturer.astype('category')\n",
        "mpg['model']=mpg.model.astype('string')\n",
        "print('--------')\n",
        "print(mpg.dtypes)"
      ],
      "id": "cell-45"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### MERGING (JOINS)"
      ],
      "id": "9935b5bf-afcb-4acf-b254-540b5090602b"
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----JOINING DATAFRAMES---\n",
            "ORIGINALS:\n",
            "    x   y   p\n",
            "0  x1  y1  p1\n",
            "1  x2  y2  p2\n",
            "2  x3  y3  p3 \n",
            "\n",
            "    x   y   w   z\n",
            "0  x1  y1  w1  z1\n",
            "1  x2  y2  w2  z2\n",
            "2  x4  y4  w4  z4"
          ]
        }
      ],
      "source": [
        "print(\"----JOINING DATAFRAMES---\")\n",
        "df1 = pd.DataFrame([['x1', 'y1','p1'], ['x2', 'y2','p2'], ['x3', 'y3','p3']], columns=['x', 'y','p'])\n",
        "df2 = pd.DataFrame([['x1', 'y1','w1', 'z1']\n",
        "                    , ['x2','y2', 'w2', 'z2']\n",
        "                    , ['x4','y4', 'w4', 'z4']],columns=['x', 'y','w', 'z'])\n",
        "print(\"ORIGINALS:\")\n",
        "print(df1,'\\n')\n",
        "print(df2)"
      ],
      "id": "cell-47"
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INNER JOIN: \n",
            "     x   y   p   w   z\n",
            "0  x1  y1  p1  w1  z1\n",
            "1  x2  y2  p2  w2  z2"
          ]
        }
      ],
      "source": [
        "print('INNER JOIN: \\n',df1.merge(df2, how='inner'))\n"
      ],
      "id": "cell-48"
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "LEFT JOIN:  \n",
            "     x   y   p    w    z\n",
            "0  x1  y1  p1   w1   z1\n",
            "1  x2  y2  p2   w2   z2\n",
            "2  x3  y3  p3  NaN  NaN"
          ]
        }
      ],
      "source": [
        "print('LEFT JOIN:  \\n',df1.merge(df2, how='left'))\n"
      ],
      "id": "cell-49"
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "RIGHT JOIN: \n",
            "     x   y    p   w   z\n",
            "0  x1  y1   p1  w1  z1\n",
            "1  x2  y2   p2  w2  z2\n",
            "2  x4  y4  NaN  w4  z4"
          ]
        }
      ],
      "source": [
        "print('RIGHT JOIN: \\n',df1.merge(df2, how='right'))\n"
      ],
      "id": "cell-50"
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "OUTER JOIN: \n",
            "     x   y    p    w    z\n",
            "0  x1  y1   p1   w1   z1\n",
            "1  x2  y2   p2   w2   z2\n",
            "2  x3  y3   p3  NaN  NaN\n",
            "3  x4  y4  NaN   w4   z4"
          ]
        }
      ],
      "source": [
        "print('OUTER JOIN: \\n',df1.merge(df2, how='outer'))\n"
      ],
      "id": "cell-51"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Concatenating series"
      ],
      "id": "0ed27223-2b3e-47ec-83bd-0c1b94ffd4f0"
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ORIGINAL\n",
            "0    a\n",
            "1    b\n",
            "dtype: object\n",
            "0    c\n",
            "1    d\n",
            "dtype: object\n",
            "OUTPUT\n",
            "0    a\n",
            "1    b\n",
            "0    c\n",
            "1    d\n",
            "dtype: object\n",
            "   0  1\n",
            "0  a  c\n",
            "1  b  d"
          ]
        }
      ],
      "source": [
        "s1 = pd.Series(['a', 'b'])\n",
        "s2 = pd.Series(['c', 'd'])\n",
        "print(\"ORIGINAL\")\n",
        "print(s1)\n",
        "print(s2)\n",
        "print(\"OUTPUT\")\n",
        "print(pd.concat([s1, s2],axis=0))\n",
        "print(pd.concat([s1, s2],axis=1))\n"
      ],
      "id": "cell-53"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Concatenating data-frames"
      ],
      "id": "625b824b-39d0-47a3-b41a-28c3c53d460b"
    },
    {
      "cell_type": "code",
      "execution_count": 43,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ORIGINAL:\n",
            "  letter  number\n",
            "0      a       1\n",
            "1      b       2\n",
            "  letter  number\n",
            "0      c       3\n",
            "1      d       4"
          ]
        }
      ],
      "source": [
        "df1 = pd.DataFrame([['a', 1], ['b', 2]], columns=['letter', 'number'])\n",
        "df2 = pd.DataFrame([['c', 3], ['d', 4]], columns=['letter', 'number'])\n",
        "print(\"ORIGINAL:\")\n",
        "print(df1)\n",
        "print(df2)"
      ],
      "id": "cell-55"
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CONCAT:\n",
            "  letter  number\n",
            "0      a       1\n",
            "1      b       2\n",
            "0      c       3\n",
            "1      d       4\n",
            "  letter  number letter  number\n",
            "0      a       1      c       3\n",
            "1      b       2      d       4"
          ]
        }
      ],
      "source": [
        "print(\"CONCAT:\")\n",
        "print(pd.concat([df1, df2],axis=0))\n",
        "print(pd.concat([df1, df2],axis=1))\n"
      ],
      "id": "cell-56"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### APPLY AND LAMBDA FUNCTIONS\n",
        "\n",
        "https://www.w3schools.com/python/python_lambda.asp\n",
        "\n",
        "A lambda function is a small anonymous function.\n",
        "\n",
        "A lambda function can take any number of arguments, but can only have\n",
        "one expression."
      ],
      "id": "abf8359b-5ccc-44d1-b981-872512c28cff"
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ORIGINAL\n",
            "   A  B\n",
            "0  1  2\n",
            "1  3  2\n",
            "ADD\n",
            " 0    3\n",
            "1    5\n",
            "dtype: int64"
          ]
        }
      ],
      "source": [
        "#iterate and apply a function along an axis of the DataFrame.\n",
        "df1 = pd.DataFrame([[1, 2], [3, 2]], columns=['A', 'B'])\n",
        "print(\"ORIGINAL\")\n",
        "print(df1)\n",
        "print(\"ADD\\n\",df1.apply(lambda x: x.A+x.B, axis=1))\n"
      ],
      "id": "cell-58"
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "EX-1: 15"
          ]
        }
      ],
      "source": [
        "f = lambda a : a + 10\n",
        "print(\"EX-1:\",f(5))"
      ],
      "id": "cell-59"
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "EX-2: 30"
          ]
        }
      ],
      "source": [
        "f = lambda a, b : a * b\n",
        "print(\"EX-2:\",f(5, 6))"
      ],
      "id": "cell-60"
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "EX-3: 33"
          ]
        }
      ],
      "source": [
        "def myfunc(n):\n",
        "    return lambda a : a * n\n",
        "mytripler = myfunc(3)\n",
        "print(\"EX-3:\",mytripler(11))"
      ],
      "id": "cell-61"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Melting and Pivoting"
      ],
      "id": "c19ee9fe-c45f-449d-b0d2-e4055984413e"
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----WIDE FORMAT----\n",
            "    ID   H   W   A\n",
            "0  ID0  H0  W0  A0\n",
            "1  ID1  H1  W1  A1\n",
            "2  ID2  H2  W2  A2"
          ]
        }
      ],
      "source": [
        "#----------------------\n",
        "#CONSIDER A SURVEY\n",
        "#----------------------\n",
        "# with 3 participants (subject) \n",
        "# and  2 measurements (HEIGHT,WEIGHT)=(H,W)\n",
        "#print(\"----------\",example,\"----------\")\n",
        "\n",
        "print(\"----WIDE FORMAT----\")\n",
        "# ONE \"OBSERVATION\" IS ONE \"PARTICIPANT\"\n",
        "df1 = pd.DataFrame(\n",
        "    {'ID': {0: 'ID0', 1: 'ID1', 2: 'ID2'},\n",
        "        'H' : {0: \"H0\" , 1: \"H1\" , 2: \"H2\"},\n",
        "        'W' : {0: \"W0\" , 1: \"W1\" , 2: \"W2\"}  ,\n",
        "        'A' : {0: \"A0\" , 1: \"A1\" , 2: \"A2\"}\n",
        "        }\n",
        "                    )\n",
        "print(df1)"
      ],
      "id": "cell-63"
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----LONG FORMAT----\n",
            "    ID variable value\n",
            "0  ID0        H    H0\n",
            "1  ID1        H    H1\n",
            "2  ID2        H    H2\n",
            "3  ID0        W    W0\n",
            "4  ID1        W    W1\n",
            "5  ID2        W    W2\n",
            "6  ID0        A    A0\n",
            "7  ID1        A    A1\n",
            "8  ID2        A    A2"
          ]
        }
      ],
      "source": [
        "   \n",
        "print(\"----LONG FORMAT----\")\n",
        "# ONE \"OBSERVATION\" IS ONE \"MEASUREMENT\" OF A PARTICIPANT\n",
        "\n",
        "#NEED TELL IT WHAT AN \"OBSERVATION\" (ID) IS\n",
        "df2=pd.melt(df1, id_vars =['ID']) \n",
        "print(df2)\n"
      ],
      "id": "cell-64"
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "MELT-1 ONLY ALONG A AND W DIMENSIONS\n",
            "    ID   H variable value\n",
            "0  ID0  H0        W    W0\n",
            "1  ID1  H1        W    W1\n",
            "2  ID2  H2        W    W2\n",
            "3  ID0  H0        A    A0\n",
            "4  ID1  H1        A    A1\n",
            "5  ID2  H2        A    A2\n",
            "MELT-2 ONLY ALONG A DIMENSION\n",
            "    ID   H   W variable value\n",
            "0  ID0  H0  W0        A    A0\n",
            "1  ID1  H1  W1        A    A1\n",
            "2  ID2  H2  W2        A    A2"
          ]
        }
      ],
      "source": [
        "print(\"MELT-1 ONLY ALONG A AND W DIMENSIONS\")\n",
        "print(pd.melt(df1, id_vars =['ID','H']))\n",
        "\n",
        "print(\"MELT-2 ONLY ALONG A DIMENSION\")\n",
        "print(pd.melt(df1, id_vars =['ID','H','W']))\n",
        "# RENAME\n",
        "# print(df.melt(id_vars=[\"ID\"], var_name = 'var-name', value_name='VALUE'))"
      ],
      "id": "cell-65"
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----WIDE FORMAT (PIVOT BACK)----\n",
            "    ID   A   H   W\n",
            "0  ID0  A0  H0  W0\n",
            "1  ID1  A1  H1  W1\n",
            "2  ID2  A2  H2  W2"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/var/folders/0l/7994y9sn0l1d6cxc7lqc26p80000gn/T/ipykernel_18079/2262497541.py:3: FutureWarning: In a future version of pandas all arguments of DataFrame.pivot will be keyword-only.\n",
            "  df2=df2.pivot(*df2).rename_axis(columns = None).reset_index()"
          ]
        }
      ],
      "source": [
        "#CONVERT BACK\n",
        "print(\"----WIDE FORMAT (PIVOT BACK)----\")\n",
        "df2=df2.pivot(*df2).rename_axis(columns = None).reset_index() \n",
        "print(df2)"
      ],
      "id": "cell-66"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Splitting"
      ],
      "id": "7f4413d3-4cf5-4570-9d74-6a004f135475"
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ORIGNIAL-0\n",
            "        A\n",
            "0   m143\n",
            "1  f1232\n",
            "MODIFIED-0\n",
            "   MF   NUM\n",
            "0  m   143\n",
            "1  f  1232"
          ]
        }
      ],
      "source": [
        "#---------------------------------\n",
        "# SPLITTING STRINGS INTO MULTIPLE COLUMN  \n",
        "#---------------------------------\n",
        "df1 = pd.DataFrame([['m143'], ['f1232']], columns=['A'])\n",
        "print(\"ORIGNIAL-0\\n\",df1)\n",
        "\n",
        "#SPLITTING m143 --> m 143\n",
        "#Series.str()-- >Vectorized string functions for Series and Index.\n",
        "df1[\"MF\"]  = df1.A.str[0]\n",
        "df1[\"NUM\"] = df1.A.str[1:]\n",
        "df1=df1.drop('A',axis=1)                                 \n",
        "print(\"MODIFIED-0\\n\",df1)"
      ],
      "id": "cell-68"
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ORIGNIAL-1\n",
            "          A       B\n",
            "0   [1, 2]  [3, 4]\n",
            "1  [NA, 6]  [7, 8]\n",
            "MODIFIED-1\n",
            "    A1  A2 B1  B2\n",
            "0   1   2  3   4\n",
            "1  NA   6  7   8"
          ]
        }
      ],
      "source": [
        "#---------------------------------\n",
        "# SPLITTING A LIST STORED AS A STRING\n",
        "#---------------------------------\n",
        "# #Series.str()-- >Vectorized string functions for Series and Index.\n",
        "df1 = pd.DataFrame([['[1, 2]','[3, 4]'], ['[NA, 6]','[7, 8]']], columns=['A', 'B'])\n",
        "print(\"ORIGNIAL-1\\n\",df1)\n",
        "\n",
        "df1[['A1','A2']] = pd.DataFrame(df1.A.str.replace('[','', regex=True).str.replace(']','', regex=True).str.split(',').tolist())\n",
        "df1[['B1','B2']] = pd.DataFrame(df1.B.str.replace('[','', regex=True).str.replace(']','', regex=True).str.split(',').tolist())\n",
        "df1=df1.drop(['A','B'],axis=1)                                 \n",
        "print(\"MODIFIED-1\\n\",df1)"
      ],
      "id": "cell-69"
    },
    {
      "cell_type": "code",
      "execution_count": 55,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ORIGNIAL-0\n",
            "        A\n",
            "0   m143\n",
            "1  f1232\n",
            "MODIFIED-0\n",
            "   MF   NUM\n",
            "0  m   143\n",
            "1  f  1232"
          ]
        }
      ],
      "source": [
        "#---------------------------------\n",
        "# SPLITTING STRINGS INTO MULTIPLE COLUMNS (ALTERNATIVE METHOD)\n",
        "#---------------------------------\n",
        "df1 = pd.DataFrame([['m143'], ['f1232']], columns=['A'])\n",
        "print(\"ORIGNIAL-0\\n\",df1)\n",
        "\n",
        "#SPLITTING m143 --> m 143\n",
        "df1 = df1.assign(                               # create new columns\n",
        "  MF    = lambda x: x.A.str[0].astype(str),\n",
        "  NUM   = lambda x: x.A.str[1:].astype(str)).drop('A', axis=1)                     # Remove old column\n",
        "print(\"MODIFIED-0\\n\",df1)\n"
      ],
      "id": "cell-70"
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ORIGNIAL-0\n",
            "        A\n",
            "0   m143\n",
            "1  f1232\n",
            "MODIFIED-0\n",
            "   MF   NUM\n",
            "0  m   143\n",
            "1  f  1232"
          ]
        }
      ],
      "source": [
        "#---------------------------------\n",
        "# #SPLITTING STRINGS INTO MULTIPLE COLUMN (ALTERNATIVE METHOD)\n",
        "#---------------------------------\n",
        "df1 = pd.DataFrame([['m143'], ['f1232']], columns=['A'])\n",
        "print(\"ORIGNIAL-0\\n\",df1)\n",
        "\n",
        "#SPLITTING m143 --> m 143\n",
        "df1 = df1.assign(                          # create new columns\n",
        "  MF    = df1.A.str[0],\n",
        "  NUM   = df1.A.str[1:],\n",
        "                ).drop('A', axis=1)        # Remove old column\n",
        "print(\"MODIFIED-0\\n\",df1)\n"
      ],
      "id": "cell-71"
    },
    {
      "cell_type": "code",
      "execution_count": 57,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ORIGNIAL-2\n",
            "         A       B\n",
            "0  [1, 2]  [3, 4]\n",
            "1  [5, 6]  [7, 8]\n",
            "MODIFIED-2\n",
            "   A1  A2 B1  B2\n",
            "0  1   2  3   4\n",
            "1  5   6  7   8"
          ]
        }
      ],
      "source": [
        "#---------------------------------\n",
        "# SPLITTING A LIST STORED AS A STRING (ALTERNATIVE METHOD)\n",
        "#---------------------------------\n",
        "df1 = pd.DataFrame([['[1, 2]','[3, 4]'], ['[5, 6]','[7, 8]']], columns=['A', 'B'])\n",
        "print(\"ORIGNIAL-2\\n\",df1)\n",
        "df1['A']  = df1.apply(lambda x: x.A.replace('[','').replace(']','').split(','), axis=1)\n",
        "df1['B']  = df1.apply(lambda x: x.B.replace('[','').replace(']','').split(','), axis=1)\n",
        "df1[['A1','A2']] = pd.DataFrame(df1.A.tolist())\n",
        "df1[['B1','B2']] = pd.DataFrame(df1.B.tolist())\n",
        "df1=df1.drop(['A','B'],axis=1)\n",
        "print(\"MODIFIED-2\\n\",df1)\n"
      ],
      "id": "cell-72"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### MISSING VALUES"
      ],
      "id": "1de5a641-69b6-4687-a18c-6d757365fa54"
    },
    {
      "cell_type": "code",
      "execution_count": 58,
      "metadata": {},
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "     A     B     C     D\n",
            "0 -2.0  -1.0   1.0   2.0\n",
            "1  1.0  -2.0   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0\n",
            "\n",
            "------INJECT MISSING DATA------\n",
            "     A     B     C     D\n",
            "0  NaN   NaN   1.0   2.0\n",
            "1  1.0   NaN   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0\n",
            "\n",
            "isnull\n",
            "        A      B      C      D\n",
            "0   True   True  False  False\n",
            "1  False   True  False  False\n",
            "2  False  False  False  False\n",
            "3  False  False  False  False\n",
            "\n",
            "isna\n",
            "        A      B      C      D\n",
            "0   True   True  False  False\n",
            "1  False   True  False  False\n",
            "2  False  False  False  False\n",
            "3  False  False  False  False\n",
            "\n",
            "drop columns\n",
            "       C     D\n",
            "0   1.0   2.0\n",
            "1   3.0   4.0\n",
            "2   7.0   8.0\n",
            "3  11.0  12.0\n",
            "\n",
            "drop rows\n",
            "      A     B     C     D\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0\n",
            "\n",
            "fill NaN with 5\n",
            "      A     B     C     D\n",
            "0  5.0   5.0   1.0   2.0\n",
            "1  1.0   5.0   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0\n",
            "\n",
            "average of col-D:  6.5\n",
            "\n",
            "fill NaN with average of col-D\n",
            "      A     B     C     D\n",
            "0  6.5   6.5   1.0   2.0\n",
            "1  1.0   6.5   3.0   4.0\n",
            "2  5.0   6.0   7.0   8.0\n",
            "3  9.0  10.0  11.0  12.0"
          ]
        }
      ],
      "source": [
        "#DATA-FRAME (FROM NP ARRAY)\n",
        "values=np.array([[-2,-1,1,2],[1,-2,3,4],[5,6,7.,8],[9,10,11,12]])\n",
        "df = pd.DataFrame(values, columns=list(\"ABCD\"))\n",
        "print(df)\n",
        "\n",
        "print(\"\\n------INJECT MISSING DATA------\")\n",
        "df1=df[df > -1]\n",
        "print(df1)\n",
        "\n",
        "print(\"\\nisnull\\n\",df1.isnull())\n",
        "print(\"\\nisna\\n\",df1.isna())\n",
        "\n",
        "# drop all columns that have missing data\n",
        "print(\"\\ndrop columns\\n\",df1.dropna(axis= 1))\n",
        "\n",
        "# #to drop any rows that have missing data.\n",
        "print(\"\\ndrop rows\\n\",df1.dropna(axis= 0))\n",
        "\n",
        "print(\"\\nfill NaN with 5\\n\",df1.fillna(value=5))\n",
        "\n",
        "print(\"\\naverage of col-D: \",df1[\"D\"].mean())\n",
        "print(\"\\nfill NaN with average of col-D\\n\",df1.fillna(df1[\"D\"].mean()))"
      ],
      "id": "cell-74"
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### EXTRA CODE"
      ],
      "id": "a210382e-eedd-42d8-acfc-dd6825f9bacc"
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "# print(\"----USING INDEX NOT ID----\")\n",
        "# print(df1)\n",
        "# print(df1.index)\n",
        "# print(df1.columns)\n",
        "\n",
        "# df2.index=df1['ID']\n",
        "# df2=df2.drop(['ID'],axis=1)\n",
        "# print(df2)\n",
        "# print(df2.index)\n",
        "# print(df2.columns)    \n",
        "# print(pd.melt(df2))\n",
        "\n",
        "# #DATES\n",
        "# print(\"----------------------\")\n",
        "# dates = pd.date_range(\"20130101\", periods=4)\n",
        "# print(dates)\n",
        "\n",
        "# #---------------\n",
        "# # OPERATIONS\n",
        "# #---------------\n",
        "# print(\"\\n------STATS------\")\n",
        "# print(df)\n",
        "# print('------------')\n",
        "# print(df.mean(0))     #axis=0 (down rowns)\n",
        "# print('------------')\n",
        "# print(df.mean(1))     #axis=1 (accross columns)\n",
        "\n",
        "\n",
        "# print(\"\\n------APPLY FUNCTIONS------\")\n",
        "# print(df)\n",
        "# print('------------')\n",
        "# print(df.apply(np.cumsum))    #CUMULATIVE SUM DOWN COLUMN\n",
        "# print('------------')\n",
        "# print(df.apply(lambda x: x.max() - x.min()))\n",
        " \n",
        "# # create a sample column\n",
        "# nums = pd.Series([\"1\", \"2\", \"3.14\", \"5\"])\n",
        "\n",
        "# # print the series\n",
        "# print(nums)\n",
        "\n",
        "# # print the data type\n",
        "# print('\\nData Type: ',nums.dtype)\n",
        "\n",
        "\n",
        "\n"
      ],
      "id": "cell-76"
    }
  ],
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3.10.4 (‘ANLY590’)",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "codemirror_mode": {
        "name": "ipython",
        "version": "3"
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.8"
    },
    "orig_nbformat": "4",
    "vscode": {
      "interpreter": {
        "hash": "2b6082c1c9eef3a910163f232074f2e179e34ed8469dd2c24c723d1290ad549e"
      }
    }
  }
}