{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.8.5"},"latex_envs":{"LaTeX_envs_menu_present":true,"autoclose":false,"autocomplete":true,"bibliofile":"biblio.bib","cite_by":"apalike","current_citInitial":1,"eqLabelWithNumbers":true,"eqNumInitial":1,"hotkeys":{"equation":"Ctrl-E","itemize":"Ctrl-I"},"labels_anchors":false,"latex_user_defs":false,"report_style_numbering":false,"user_envs_cfg":false}},"cells":[{"cell_type":"code","metadata":{"id":"VYngLso0JUg_","executionInfo":{"status":"ok","timestamp":1706703061795,"user_tz":-60,"elapsed":23883,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}}},"source":["!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n","!wget -q http://archive.apache.org/dist/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz\n","!tar xf spark-3.1.1-bin-hadoop3.2.tgz\n","!pip install -q findspark"],"execution_count":1,"outputs":[]},{"cell_type":"code","metadata":{"id":"UsnMvQDRIG8p","executionInfo":{"status":"ok","timestamp":1706703067396,"user_tz":-60,"elapsed":352,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}}},"source":["import os\n","os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n","os.environ[\"SPARK_HOME\"] = \"/content/spark-3.1.1-bin-hadoop3.2\""],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"id":"evOnjggtJEXg","executionInfo":{"status":"ok","timestamp":1706703079417,"user_tz":-60,"elapsed":9445,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"colab":{"base_uri":"https://localhost:8080/","height":219},"outputId":"e97cf97f-3e84-4eae-c709-9ab9c6312034"},"source":["import findspark\n","findspark.init()\n","from pyspark.sql import SparkSession\n","spark = SparkSession.builder.master(\"local[*]\").getOrCreate()\n","spark.conf.set(\"spark.sql.repl.eagerEval.enabled\", True) # Property used to format output tables better\n","spark"],"execution_count":3,"outputs":[{"output_type":"execute_result","data":{"text/plain":["<pyspark.sql.session.SparkSession at 0x783a4e7015a0>"],"text/html":["\n","            <div>\n","                <p><b>SparkSession - in-memory</b></p>\n","                \n","        <div>\n","            <p><b>SparkContext</b></p>\n","\n","            <p><a href=\"http://f42a23bf57af:4040\">Spark UI</a></p>\n","\n","            <dl>\n","              <dt>Version</dt>\n","                <dd><code>v3.1.1</code></dd>\n","              <dt>Master</dt>\n","                <dd><code>local[*]</code></dd>\n","              <dt>AppName</dt>\n","                <dd><code>pyspark-shell</code></dd>\n","            </dl>\n","        </div>\n","        \n","            </div>\n","        "]},"metadata":{},"execution_count":3}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"0LSsYt0-I7J7","executionInfo":{"status":"ok","timestamp":1706703095772,"user_tz":-60,"elapsed":818,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"75c9e771-524e-40b2-8c95-3f46f8a44541"},"source":["import pyspark\n","type(spark)"],"execution_count":4,"outputs":[{"output_type":"execute_result","data":{"text/plain":["pyspark.sql.session.SparkSession"]},"metadata":{},"execution_count":4}]},{"cell_type":"code","metadata":{"id":"6J2Cm3DMJZkZ","executionInfo":{"status":"ok","timestamp":1706703098934,"user_tz":-60,"elapsed":688,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}}},"source":["sc = spark.sparkContext"],"execution_count":5,"outputs":[]},{"cell_type":"code","metadata":{"id":"SB-1ZpL9P9ve","executionInfo":{"status":"ok","timestamp":1706710830803,"user_tz":-60,"elapsed":236,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}}},"source":["s = '''Every person had a star, every star had a friend, and for every person\n","carrying a star there was someone else who reflected it, and everyone\n","carried this reflection like a secret confidante in the heart'''\n","simple_rdd = sc.parallelize(s.split('\\n'))"],"execution_count":40,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"pwq1i1gGp7lz","executionInfo":{"status":"ok","timestamp":1706710831452,"user_tz":-60,"elapsed":3,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"acb4e0ff-5bce-4887-e246-86fff2255a9a"},"source":["simple_rdd"],"execution_count":41,"outputs":[{"output_type":"execute_result","data":{"text/plain":["ParallelCollectionRDD[112] at readRDDFromFile at PythonRDD.scala:274"]},"metadata":{},"execution_count":41}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"8pe238QyJsCP","executionInfo":{"status":"ok","timestamp":1706710832228,"user_tz":-60,"elapsed":3,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"60fcb339-4f3d-4f3a-df03-3b74a6a89bbd"},"source":["simple_rdd.collect()"],"execution_count":42,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['Every person had a star, every star had a friend, and for every person',\n"," 'carrying a star there was someone else who reflected it, and everyone',\n"," 'carried this reflection like a secret confidante in the heart']"]},"metadata":{},"execution_count":42}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"DMdhlgkPVAu1","executionInfo":{"status":"ok","timestamp":1706710833146,"user_tz":-60,"elapsed":4,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"78cce61b-057a-49e3-d18b-8291ceb6ee73"},"source":["simple_rdd.map(lambda line: line.split(' '))"],"execution_count":43,"outputs":[{"output_type":"execute_result","data":{"text/plain":["PythonRDD[113] at RDD at PythonRDD.scala:53"]},"metadata":{},"execution_count":43}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"LymNHuZDQlgZ","executionInfo":{"status":"ok","timestamp":1706710834160,"user_tz":-60,"elapsed":336,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"3ca6abe5-82e2-4292-9bdf-3e77a8091c0a"},"source":["(simple_rdd.map(lambda line: line.split(' '))\n","           .collect())"],"execution_count":44,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[['Every',\n","  'person',\n","  'had',\n","  'a',\n","  'star,',\n","  'every',\n","  'star',\n","  'had',\n","  'a',\n","  'friend,',\n","  'and',\n","  'for',\n","  'every',\n","  'person'],\n"," ['carrying',\n","  'a',\n","  'star',\n","  'there',\n","  'was',\n","  'someone',\n","  'else',\n","  'who',\n","  'reflected',\n","  'it,',\n","  'and',\n","  'everyone'],\n"," ['carried',\n","  'this',\n","  'reflection',\n","  'like',\n","  'a',\n","  'secret',\n","  'confidante',\n","  'in',\n","  'the',\n","  'heart']]"]},"metadata":{},"execution_count":44}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"OGbH_xQyQ1pJ","executionInfo":{"status":"ok","timestamp":1706710835885,"user_tz":-60,"elapsed":575,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"d3052d41-70c7-415e-ec7c-c090cb3f4b9f"},"source":["(simple_rdd.flatMap(lambda line: line.split(' '))\n","           .take(17))"],"execution_count":45,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['Every',\n"," 'person',\n"," 'had',\n"," 'a',\n"," 'star,',\n"," 'every',\n"," 'star',\n"," 'had',\n"," 'a',\n"," 'friend,',\n"," 'and',\n"," 'for',\n"," 'every',\n"," 'person',\n"," 'carrying',\n"," 'a',\n"," 'star']"]},"metadata":{},"execution_count":45}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"PsArbQfrRH2l","executionInfo":{"status":"ok","timestamp":1706710876036,"user_tz":-60,"elapsed":322,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"4dca06b2-a241-4033-ebed-ac8e4c0243ac"},"source":["(simple_rdd.flatMap(lambda line: line.split(' '))\n","           .map(lambda word: word.replace(',', '').lower())\n","           .take(10))"],"execution_count":46,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['every', 'person', 'had', 'a', 'star', 'every', 'star', 'had', 'a', 'friend']"]},"metadata":{},"execution_count":46}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"mp8z9tJNRVQZ","executionInfo":{"status":"ok","timestamp":1706703109119,"user_tz":-60,"elapsed":5,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"3d1b35b3-b2b9-4d2d-8d0f-92476c4cb232"},"source":["(simple_rdd.flatMap(lambda line: line.split(' '))\n","           .map(lambda word: word.replace(',', '').lower())\n","           .map(lambda word: (word, 1))\n","           .take(5))"],"execution_count":13,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[('every', 1), ('person', 1), ('had', 1), ('a', 1), ('star', 1)]"]},"metadata":{},"execution_count":13}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"-Vcv7UcsRjBr","executionInfo":{"status":"ok","timestamp":1706703110718,"user_tz":-60,"elapsed":1602,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"9658f63d-f56d-4826-fe80-569bee8ee9d8"},"source":["(simple_rdd.flatMap(lambda line: line.split(' '))\n","           .map(lambda word: word.replace(',', '').lower())\n","           .map(lambda word: (word, 1))\n","           .reduceByKey(lambda a,b: a+b)\n","           .collect())"],"execution_count":14,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[('person', 2),\n"," ('there', 1),\n"," ('was', 1),\n"," ('carried', 1),\n"," ('this', 1),\n"," ('like', 1),\n"," ('secret', 1),\n"," ('confidante', 1),\n"," ('in', 1),\n"," ('heart', 1),\n"," ('every', 3),\n"," ('had', 2),\n"," ('a', 4),\n"," ('star', 3),\n"," ('friend', 1),\n"," ('and', 2),\n"," ('for', 1),\n"," ('carrying', 1),\n"," ('someone', 1),\n"," ('else', 1),\n"," ('who', 1),\n"," ('reflected', 1),\n"," ('it', 1),\n"," ('everyone', 1),\n"," ('reflection', 1),\n"," ('the', 1)]"]},"metadata":{},"execution_count":14}]},{"cell_type":"code","metadata":{"id":"70Zlz-SER3ki","executionInfo":{"status":"ok","timestamp":1706703110719,"user_tz":-60,"elapsed":6,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}}},"source":["def count_freq(rdd):\n","  return (rdd.flatMap(lambda line: line.split(' '))\n","            .map(lambda word: word.replace(',', '').lower())\n","            .map(lambda word: (word, 1))\n","            .reduceByKey(lambda a,b: a+b)\n","            .sortBy(lambda a: a[1], False)\n","            .collect())"],"execution_count":15,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"dA_3tetsHQm8","executionInfo":{"status":"ok","timestamp":1706703111088,"user_tz":-60,"elapsed":374,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"aa28a984-1582-40b9-f7ca-6e150bfad962"},"source":["!wget http://www.scifiscripts.com/scripts/swd1_5-74.txt"],"execution_count":16,"outputs":[{"output_type":"stream","name":"stdout","text":["--2024-01-31 12:11:50--  http://www.scifiscripts.com/scripts/swd1_5-74.txt\n","Resolving www.scifiscripts.com (www.scifiscripts.com)... 207.32.177.145\n","Connecting to www.scifiscripts.com (www.scifiscripts.com)|207.32.177.145|:80... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 203125 (198K) [text/plain]\n","Saving to: ‘swd1_5-74.txt’\n","\n","\rswd1_5-74.txt         0%[                    ]       0  --.-KB/s               \rswd1_5-74.txt       100%[===================>] 198.36K  --.-KB/s    in 0.08s   \n","\n","2024-01-31 12:11:50 (2.43 MB/s) - ‘swd1_5-74.txt’ saved [203125/203125]\n","\n"]}]},{"cell_type":"code","metadata":{"id":"EZA0B20IJxkN","executionInfo":{"status":"ok","timestamp":1706703111088,"user_tz":-60,"elapsed":3,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}}},"source":["sw = sc.textFile('swd1_5-74.txt')"],"execution_count":17,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"VjXWVqhOJ1Vu","executionInfo":{"status":"ok","timestamp":1706703111657,"user_tz":-60,"elapsed":260,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"dc11b999-c3ea-41f6-9fc2-6bc36873107b"},"source":["sw.take(10)"],"execution_count":18,"outputs":[{"output_type":"execute_result","data":{"text/plain":["['The Star Wars',\n"," 'by',\n"," 'George Lucas',\n"," '',\n"," '',\n"," '',\n"," 'Rough Draft [First of four major screenplay drafts]',\n"," 'Lucasfilm Ltd.',\n"," '5/74',\n"," '']"]},"metadata":{},"execution_count":18}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"GjY7hco0J6Mu","executionInfo":{"status":"ok","timestamp":1706703113528,"user_tz":-60,"elapsed":1873,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"0539a417-027b-4444-d5ea-5cc02c975d95"},"source":["count_freq(sw)[: 100]"],"execution_count":19,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[('', 3543),\n"," ('the', 2731),\n"," ('and', 929),\n"," ('a', 897),\n"," ('of', 744),\n"," ('to', 702),\n"," ('general', 459),\n"," ('is', 393),\n"," ('in', 359),\n"," ('-', 330),\n"," ('starkiller', 302),\n"," ('his', 289),\n"," ('on', 274),\n"," ('he', 243),\n"," ('are', 209),\n"," ('you', 197),\n"," ('with', 196),\n"," ('they', 177),\n"," ('as', 162),\n"," ('two', 161),\n"," ('into', 153),\n"," ('for', 146),\n"," ('it', 145),\n"," ('be', 145),\n"," ('han', 143),\n"," ('at', 137),\n"," ('i', 134),\n"," ('from', 134),\n"," ('out', 129),\n"," ('by', 125),\n"," ('an', 122),\n"," ('whitsun', 122),\n"," ('have', 119),\n"," ('one', 114),\n"," ('aquilae', 114),\n"," ('your', 106),\n"," ('but', 101),\n"," ('all', 95),\n"," ('we', 94),\n"," ('threepio', 92),\n"," ('through', 84),\n"," ('back', 83),\n"," ('princess', 82),\n"," ('this', 80),\n"," ('artwo', 80),\n"," ('their', 79),\n"," ('will', 79),\n"," ('small', 77),\n"," ('him', 75),\n"," ('up', 74),\n"," ('get', 73),\n"," ('fortress', 72),\n"," ('them', 71),\n"," ('there', 69),\n"," ('imperial', 69),\n"," ('down', 67),\n"," ('space', 66),\n"," ('over', 66),\n"," ('large', 65),\n"," (\"it's\", 64),\n"," ('then', 63),\n"," (\"i'm\", 63),\n"," ('not', 59),\n"," ('young', 58),\n"," ('before', 57),\n"," ('that', 55),\n"," ('captain', 55),\n"," ('has', 54),\n"," ('off', 54),\n"," ('around', 53),\n"," ('no', 53),\n"," ('starship', 53),\n"," ('little', 53),\n"," ('pilot', 53),\n"," ('her', 52),\n"," ('valorum', 52),\n"," ('my', 52),\n"," ('main', 50),\n"," ('can', 50),\n"," ('way', 49),\n"," ('troops', 49),\n"," ('several', 47),\n"," ('must', 47),\n"," ('officer', 47),\n"," ('leia', 46),\n"," ('biggs', 45),\n"," ('kane', 44),\n"," ('power', 44),\n"," ('three', 44),\n"," ('been', 44),\n"," ('going', 43),\n"," ('what', 43),\n"," ('yavin', 43),\n"," ('jedi', 42),\n"," ('old', 42),\n"," ('vader', 42),\n"," ('annikin', 41),\n"," ('turns', 41),\n"," ('room', 41),\n"," (\"we're\", 41)]"]},"metadata":{},"execution_count":19}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"SRPuxtCuSHOH","executionInfo":{"status":"ok","timestamp":1706703120801,"user_tz":-60,"elapsed":7277,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"4cd9e03c-4416-445b-c711-0926b6afbf79"},"source":["import random\n","\n","NUM_SAMPLES = 10**7\n","\n","def inside(p):\n","    x, y = random.random(), random.random()\n","    return x*x + y*y < 1\n","\n","count = (sc.parallelize(range(0, NUM_SAMPLES))\n","           .filter(inside).count())\n","\n","print('Pi is roughly {}'.format(4.0 * count / NUM_SAMPLES))"],"execution_count":20,"outputs":[{"output_type":"stream","name":"stdout","text":["Pi is roughly 3.1417288\n"]}]},{"cell_type":"code","source":["!wget https://jacobceles.github.io/knowledge_repo/colab_and_pyspark/cars.csv"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"q9KNpewCndUO","executionInfo":{"status":"ok","timestamp":1706706198366,"user_tz":-60,"elapsed":656,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"20853ff7-36ee-4c2d-c6b3-b8b9c392589b"},"execution_count":21,"outputs":[{"output_type":"stream","name":"stdout","text":["--2024-01-31 13:03:17--  https://jacobceles.github.io/knowledge_repo/colab_and_pyspark/cars.csv\n","Resolving jacobceles.github.io (jacobceles.github.io)... 185.199.110.153, 185.199.111.153, 185.199.108.153, ...\n","Connecting to jacobceles.github.io (jacobceles.github.io)|185.199.110.153|:443... connected.\n","HTTP request sent, awaiting response... 301 Moved Permanently\n","Location: https://jacobcelestine.com/knowledge_repo/colab_and_pyspark/cars.csv [following]\n","--2024-01-31 13:03:17--  https://jacobcelestine.com/knowledge_repo/colab_and_pyspark/cars.csv\n","Resolving jacobcelestine.com (jacobcelestine.com)... 185.199.108.153, 185.199.109.153, 185.199.110.153, ...\n","Connecting to jacobcelestine.com (jacobcelestine.com)|185.199.108.153|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 22608 (22K) [text/csv]\n","Saving to: ‘cars.csv’\n","\n","\rcars.csv              0%[                    ]       0  --.-KB/s               \rcars.csv            100%[===================>]  22.08K  --.-KB/s    in 0.001s  \n","\n","2024-01-31 13:03:17 (20.2 MB/s) - ‘cars.csv’ saved [22608/22608]\n","\n"]}]},{"cell_type":"code","metadata":{"id":"hlKe-wRE6fRX","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1706706592314,"user_tz":-60,"elapsed":845,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"3d765be0-1051-42d2-9dcb-48b657f15154"},"source":["cars = spark.read.csv('cars.csv', header=True, sep=\";\")\n","cars.show(5)"],"execution_count":32,"outputs":[{"output_type":"stream","name":"stdout","text":["+--------------------+----+---------+------------+----------+------+------------+-----+------+\n","|                 Car| MPG|Cylinders|Displacement|Horsepower|Weight|Acceleration|Model|Origin|\n","+--------------------+----+---------+------------+----------+------+------------+-----+------+\n","|Chevrolet Chevell...|18.0|        8|       307.0|     130.0| 3504.|        12.0|   70|    US|\n","|   Buick Skylark 320|15.0|        8|       350.0|     165.0| 3693.|        11.5|   70|    US|\n","|  Plymouth Satellite|18.0|        8|       318.0|     150.0| 3436.|        11.0|   70|    US|\n","|       AMC Rebel SST|16.0|        8|       304.0|     150.0| 3433.|        12.0|   70|    US|\n","|         Ford Torino|17.0|        8|       302.0|     140.0| 3449.|        10.5|   70|    US|\n","+--------------------+----+---------+------------+----------+------+------------+-----+------+\n","only showing top 5 rows\n","\n"]}]},{"cell_type":"code","source":["cars.select('Cylinders').rdd.map(lambda c: (c, 1)).reduceByKey(lambda a, b: a+b).collect()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"HqVIMyAPn3sx","executionInfo":{"status":"ok","timestamp":1706706634477,"user_tz":-60,"elapsed":619,"user":{"displayName":"Dario Malchiodi","userId":"00228704477903575446"}},"outputId":"4b25711e-8ad2-465f-b007-eb395fc763f9"},"execution_count":37,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[(Row(Cylinders='8'), 108),\n"," (Row(Cylinders='4'), 207),\n"," (Row(Cylinders='6'), 84),\n"," (Row(Cylinders='3'), 4),\n"," (Row(Cylinders='5'), 3)]"]},"metadata":{},"execution_count":37}]},{"cell_type":"code","source":[],"metadata":{"id":"zHORYwC1og9Z"},"execution_count":null,"outputs":[]}]}