title: error while defining SparkContext in jupyter notebooke(Python 3 (ipykernel))
body: <p><a href="https://i.stack.imgur.com/ouNha.png" rel="nofollow noreferrer">sc = SparkContext(conf=conf) error</a></p>
  <p>using anaconda`s Jupyter notebook, this is my first time using pyspark.
  what I am trying to do is defining the spark environment to read a csv file on the local disk.</p>
  <p>Edit: added the text as user2314737 asked for text instead of an image and now StackOverflow wants me to add more details as my text body Is mostly code XD so i have to chat with you all in this part you can ignore this text body, but i do have to write until i can press save edits.</p>
  <p>Input:</p>
  <pre class="lang-py prettyprint-override"><code>
  from pyspark import SparkContext, SparkConf
  conf = SparkConf().setAppName(&quot;PrdectiveModel&quot;)
  sc = SparkContext(conf=conf) ----&gt; the error is in this line
  </code></pre>
  <p>Output:</p>
  <pre class="lang-py prettyprint-override"><code>Py4JJavaError                             Traceback (most recent call last)
  Input In [13], in &lt;cell line: 3&gt;()
        1 from pyspark import SparkContext, SparkConf
        2 conf = SparkConf().setAppName(&quot;PrdectiveModel&quot;)
  ----&gt; 3 sc = SparkContext(conf=conf)
  
  File E:\anaconda3\lib\site-packages\pyspark\context.py:146, in SparkContext.__init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)
      144 SparkContext._ensure_initialized(self, gateway=gateway, conf=conf)
      145 try:
  --&gt; 146     self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,
      147                   conf, jsc, profiler_cls)
      148 except:
      149     # If an error occurs, clean up in order to allow future SparkContext creation:
      150     self.stop()
  
  File E:\anaconda3\lib\site-packages\pyspark\context.py:209, in SparkContext._do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls)
      206 self.environment[&quot;PYTHONHASHSEED&quot;] = os.environ.get(&quot;PYTHONHASHSEED&quot;, &quot;0&quot;)
      208 # Create the Java SparkContext through Py4J
  --&gt; 209 self._jsc = jsc or self._initialize_context(self._conf._jconf)
      210 # Reset the SparkConf to the one actually used by the SparkContext in JVM.
      211 self._conf = SparkConf(_jconf=self._jsc.sc().conf())
  
  File E:\anaconda3\lib\site-packages\pyspark\context.py:329, in SparkContext._initialize_context(self, jconf)
      325 def _initialize_context(self, jconf):
      326     &quot;&quot;&quot;
      327     Initialize SparkContext in function to allow subclass specific initialization
      328     &quot;&quot;&quot;
  --&gt; 329     return self._jvm.JavaSparkContext(jconf)
  
  File E:\anaconda3\lib\site-packages\py4j\java_gateway.py:1585, in JavaClass.__call__(self, *args)
     1579 command = proto.CONSTRUCTOR_COMMAND_NAME +\
     1580     self._command_header +\
     1581     args_command +\
     1582     proto.END_COMMAND_PART
     1584 answer = self._gateway_client.send_command(command)
  -&gt; 1585 return_value = get_return_value(
     1586     answer, self._gateway_client, None, self._fqn)
     1588 for temp_arg in temp_args:
     1589     temp_arg._detach()
  
  File E:\anaconda3\lib\site-packages\py4j\protocol.py:326, in get_return_value(answer, gateway_client, target_id, name)
      324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client)
      325 if answer[1] == REFERENCE_TYPE:
  --&gt; 326     raise Py4JJavaError(
      327         &quot;An error occurred while calling {0}{1}{2}.\n&quot;.
      328         format(target_id, &quot;.&quot;, name), value)
      329 else:
      330     raise Py4JError(
      331         &quot;An error occurred while calling {0}{1}{2}. Trace:\n{3}\n&quot;.
      332         format(target_id, &quot;.&quot;, name, value))
  
  Py4JJavaError: An error occurred while calling None.org.apache.spark.api.java.JavaSparkContext.
  : java.lang.NoClassDefFoundError: Could not initialize class org.apache.spark.storage.StorageUtils$
      at org.apache.spark.storage.BlockManagerMasterEndpoint.&lt;init&gt;(BlockManagerMasterEndpoint.scala:110)
      at org.apache.spark.SparkEnv$.$anonfun$create$9(SparkEnv.scala:348)
      at org.apache.spark.SparkEnv$.registerOrLookupEndpoint$1(SparkEnv.scala:287)
      at org.apache.spark.SparkEnv$.create(SparkEnv.scala:336)
      at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:191)
      at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277)
      at org.apache.spark.SparkContext.&lt;init&gt;(SparkContext.scala:460)
      at org.apache.spark.api.java.JavaSparkContext.&lt;init&gt;(JavaSparkContext.scala:58)
      at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
      at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77)
      at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
      at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)
      at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)
      at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247)
      at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
      at py4j.Gateway.invoke(Gateway.java:238)
      at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)
      at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)
      at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
      at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
      at java.base/java.lang.Thread.run(Thread.java:833)
  </code></pre>
  <p>so I did as you have said checked the environment and such.
  even used findsprak</p>
  <p>input</p>
  <pre><code>import findspark
  
  findspark.init()
  findspark.find()
  </code></pre>
  <p>output</p>
  <pre><code>'C:\\spark-3.2.1-bin-hadoop3.2'
  </code></pre>
  <p>so it is correctly installed and can be imported without any problems.</p>
  <p>YET!!!!!!!!!!!!! my error still remains</p>
  <pre><code>sc = SparkContext(conf=conf)
  </code></pre>
  <p>why does this single line cause the error</p>
  <p>..............</p>
  <p>So I tried a different code</p>
  <pre><code>import pyspark
  
  from pyspark.sql import SparkSession
  
  spark = SparkSession.builder.getOrCreate()
  
  df = spark.sql(&quot;select 'spark' as hello &quot;)
  
  df.show()
  </code></pre>
  <p>and now a different error appears</p>
  <pre><code>iRuntimeError                              Traceback (most recent call last)
  Input In [6], in &lt;cell line: 5&gt;()
        1 import pyspark
        3 from pyspark.sql import SparkSession
  ----&gt; 5 spark = SparkSession.builder.getOrCreate()
        7 df = spark.sql(&quot;select 'spark' as hello &quot;)
        9 df.show()
  
  File C:\spark-3.2.1-bin-hadoop3.2\python\pyspark\sql\session.py:228, in SparkSession.Builder.getOrCreate(self)
      226         sparkConf.set(key, value)
      227     # This SparkContext may be an existing one.
  --&gt; 228     sc = SparkContext.getOrCreate(sparkConf)
      229 # Do not update `SparkConf` for existing `SparkContext`, as it's shared
      230 # by all sessions.
      231 session = SparkSession(sc)
  
  File C:\spark-3.2.1-bin-hadoop3.2\python\pyspark\context.py:392, in SparkContext.getOrCreate(cls, conf)
      390 with SparkContext._lock:
      391     if SparkContext._active_spark_context is None:
  --&gt; 392         SparkContext(conf=conf or SparkConf())
      393     return SparkContext._active_spark_context
  
  File C:\spark-3.2.1-bin-hadoop3.2\python\pyspark\context.py:144, in SparkContext.__init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)
      139 if gateway is not None and gateway.gateway_parameters.auth_token is None:
      140     raise ValueError(
      141         &quot;You are trying to pass an insecure Py4j gateway to Spark. This&quot;
      142         &quot; is not allowed as it is a security risk.&quot;)
  --&gt; 144 SparkContext._ensure_initialized(self, gateway=gateway, conf=conf)
      145 try:
      146     self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,
      147                   conf, jsc, profiler_cls)
  
  File C:\spark-3.2.1-bin-hadoop3.2\python\pyspark\context.py:339, in SparkContext._ensure_initialized(cls, instance, gateway, conf)
      337 with SparkContext._lock:
      338     if not SparkContext._gateway:
  --&gt; 339         SparkContext._gateway = gateway or launch_gateway(conf)
      340         SparkContext._jvm = SparkContext._gateway.jvm
      342     if instance:
  
  File C:\spark-3.2.1-bin-hadoop3.2\python\pyspark\java_gateway.py:108, in launch_gateway(conf, popen_kwargs)
      105     time.sleep(0.1)
      107 if not os.path.isfile(conn_info_file):
  --&gt; 108     raise RuntimeError(&quot;Java gateway process exited before sending its port number&quot;)
      110 with open(conn_info_file, &quot;rb&quot;) as info:
      111     gateway_port = read_int(info)
  
  RuntimeError: Java gateway process exited before sending its port number
  </code></pre>
  <p>hope there is a relation to the two errors</p>
  <p>Thanks for taking your time reading this, hope it gets solved</p>