02-Create-SparkContext.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9b42a8f0-9a63-461c-95dc-27847f5b0a40",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the PySpark environment variables\n",
    "import os\n",
    "os.environ['SPARK_HOME'] = \"/Users/coder2j/Apps/Spark\"\n",
    "os.environ['PYSPARK_DRIVER_PYTHON'] = 'jupyter'\n",
    "os.environ['PYSPARK_DRIVER_PYTHON_OPTS'] = 'lab'\n",
    "os.environ['PYSPARK_PYTHON'] = 'python'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f8e53e1-7a47-4dc4-9450-6be3c466f0b3",
   "metadata": {},
   "source": [
    "## Create SparkContext in Apache Spark version 1.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "287412ee-4384-4621-b7c0-ff50013f9785",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark import SparkContext\n",
    "\n",
    "# Create a SparkContext object\n",
    "sc = SparkContext(appName=\"MySparkApplication\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "78f5b680-6bc0-42ef-99f6-83fd0da5e729",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://imac.fritz.box:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.4.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>MySparkApplication</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<SparkContext master=local[*] appName=MySparkApplication>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c57fa548-9522-4e51-8c2d-4c6937eb3b3e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shut down the current active SparkContext\n",
    "sc.stop()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6cbd0d0d-2d17-44b9-ae3b-76b413b760c3",
   "metadata": {},
   "source": [
    "## Create SparkContext in Apache Spark version 2.x and later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0508898d-8b48-4b16-bae9-87cead0489b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "# Create a SparkSession\n",
    "spark = SparkSession.builder \\\n",
    "    .appName(\"MySparkApplication\") \\\n",
    "    .getOrCreate()\n",
    "\n",
    "# Get the SparkContext from the SparkSession\n",
    "sc = spark.sparkContext\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "82c7ca1e-dd73-4306-ab7f-01bc371eb94e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://imac.fritz.box:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v3.4.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[*]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>MySparkApplication</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<SparkContext master=local[*] appName=MySparkApplication>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9a3361a7-cb9d-49a3-b80b-820e490711e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Shut down the current active SparkContext\n",
    "sc.stop() #or spark.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10336d77-bc36-4101-8055-dd7a2496d4da",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}