In [3]:
# Cell 0: Imports and Setup
import sys, os, subprocess, platform
from typing import Optional, Tuple
from pyspark.sql import DataFrame, SparkSession, functions as F, types as T, Window
from pyspark.sql.functions import col, to_date, to_timestamp
import time
# Check Python
print(f"Python: {sys.version}")
# Install psutil if needed
try:
import psutil
except Exception:
subprocess.check_call([sys.executable, "-m", "pip", "install", "psutil"])
import psutil
print("✅ All imports loaded")
Python: 3.10.18 (main, Jun 5 2025, 13:14:17) [GCC 11.2.0] ✅ All imports loaded
In [6]:
# Cell 1: Initialize Spark Session
import findspark
findspark.init()
py = sys.executable
os.environ["PYSPARK_DRIVER_PYTHON"] = py
os.environ["PYSPARK_PYTHON"] = py
spark = SparkSession.getActiveSession() or (
SparkSession.builder
.appName("Lab2-ETL")
.master("local[*]")
.config("spark.driver.memory", "8g")
.config("spark.sql.shuffle.partitions", "200")
.config("spark.sql.adaptive.enabled", "true")
.config("spark.pyspark.driver.python", py)
.config("spark.pyspark.python", py)
.getOrCreate()
)
print(f"✅ Spark {spark.version} initialized")
print(f"Master: {spark.sparkContext.master}")
WARNING: Using incubator modules: jdk.incubator.vector Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties 25/12/29 10:06:33 WARN Utils: Your hostname, Wandaogo, resolves to a loopback address: 127.0.1.1; using 10.255.255.254 instead (on interface lo) 25/12/29 10:06:33 WARN Utils: Set SPARK_LOCAL_IP if you need to bind to another address Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties Setting default log level to "WARN". To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel). 25/12/29 10:06:35 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
✅ Spark 4.0.1 initialized Master: local[*]
In [7]:
from typing import Optional, Tuple
from pyspark.sql import DataFrame, SparkSession, functions as F, types as T
def ingest(spark, path_a: str, path_b: str) -> Tuple[DataFrame, DataFrame]:
"""Charge SOURCE_A et SOURCE_B avec schémas explicites."""
print("✅ INGEST: Chargement des sources...")
events_schema = T.StructType([
T.StructField("event_time", T.TimestampType(), True),
T.StructField("event_type", T.StringType(), True),
T.StructField("session_id", T.StringType(), True),
T.StructField("product_id", T.StringType(), True),
T.StructField("price", T.DoubleType(), True),
])
sessions_schema = T.StructType([
T.StructField("session_id", T.StringType(), True),
T.StructField("user_id", T.StringType(), True),
])
df_events = (spark.read
.schema(events_schema)
.option("header", "true")
.csv(path_a))
df_sessions = (spark.read
.schema(sessions_schema)
.option("header", "true")
.csv(path_b))
events_count = df_events.count()
sessions_count = df_sessions.count()
print(f" Events loaded: {events_count} rows")
print(f" Sessions loaded: {sessions_count} rows")
malformed_events = df_events.filter(
F.col("event_time").isNull() | F.col("session_id").isNull()
).count()
print(f" Malformed events: {malformed_events}")
return df_events, df_sessions
def transform(df_a: DataFrame, df_b: DataFrame) -> DataFrame:
"""Nettoie, déduplique et normalise."""
print("\n✅ TRANSFORM: Nettoyage et normalisation...")
df_events_clean = (df_a
.withColumn("event_time", F.to_timestamp(F.col("event_time")))
.withColumn("event_date", F.to_date(F.col("event_time")))
.withColumn("price", F.col("price").cast("double"))
.filter(F.col("event_time").isNotNull())
.filter(F.col("session_id").isNotNull())
.filter(F.col("product_id").isNotNull())
.filter((F.col("price").isNull()) | (F.col("price") >= 0))
)
df_events_dedup = (df_events_clean
.withColumn("rn", F.row_number()
.over(Window.partitionBy("session_id", "product_id")
.orderBy("event_time")))
.filter(F.col("rn") == 1)
.drop("rn")
)
df_sessions_clean = (df_b
.withColumn("session_id", F.trim(F.col("session_id")))
.withColumn("user_id", F.trim(F.col("user_id")))
.filter(F.col("session_id").isNotNull())
.filter(F.col("user_id").isNotNull())
)
df_transformed = (df_events_dedup
.join(df_sessions_clean, on="session_id", how="left")
)
count_clean = df_transformed.count()
print(f" Events after transform: {count_clean} rows")
return df_transformed
def join_and_aggregate(df: DataFrame, dim: DataFrame) -> DataFrame:
"""Jointure avec table de dimension."""
print("\n✅ JOIN & AGGREGATE: Jointure et agrégations...")
from pyspark.sql.functions import broadcast
df_joined = (df
.join(broadcast(dim), on="product_id", how="left")
)
df_agg = (df_joined
.groupBy("event_date", "category", "brand")
.agg(
F.count("*").alias("event_count"),
F.sum("price").alias("total_revenue"),
F.avg("price").alias("avg_price"),
F.countDistinct("session_id").alias("unique_sessions"),
F.countDistinct("user_id").alias("unique_users")
)
.orderBy(F.desc("total_revenue"))
)
count_agg = df_agg.count()
print(f" Aggregated rows: {count_agg} rows")
return df_agg
def write_out(df: DataFrame, base: str, partitions: list) -> None:
"""Écrit Parquet en mode overwrite."""
print(f"\n✅ WRITE_OUT: Écriture vers {base}...")
import shutil
os.makedirs(base, exist_ok=True)
output_path = f"{base}/data"
if os.path.exists(output_path):
shutil.rmtree(output_path)
num_partitions = len(partitions) if partitions else 1
(df
.coalesce(num_partitions)
.write
.mode("overwrite")
.partitionBy(*partitions) if partitions else df.write.mode("overwrite")
.parquet(output_path)
)
print(f" ✅ Écrit vers: {output_path}")
print(f" Partitionné par: {partitions}")
print("✅ Pipeline API functions defined")
✅ Pipeline API functions defined
In [8]:
# Cell: PostgreSQL Configuration
import os
# Configure environment variables
os.environ['PGHOST'] = '127.0.0.1' # Force TCP (not Unix socket)
os.environ['PGPORT'] = '5433'
os.environ['PGUSER'] = 'esiee_reader'
os.environ['PGPASSWORD'] = 'azerty123'
print("✅ PostgreSQL environment configured")
print(f" Host: {os.environ['PGHOST']}")
print(f" Port: {os.environ['PGPORT']}")
print(f" User: {os.environ['PGUSER']}")
✅ PostgreSQL environment configured Host: 127.0.0.1 Port: 5433 User: esiee_reader
In [9]:
# Cell: Querying Operational Database - Assignment 2
import subprocess
import pandas as pd
print("\n" + "=" * 80)
print("🔍 ASSIGNMENT 2: QUERYING THE OPERATIONAL DATABASE")
print("=" * 80)
def run_psql_query(query: str, description: str = "") -> str:
"""Execute psql query and return result"""
if description:
print(f"\n✅ {description}")
try:
result = subprocess.run(
["psql", "esiee_full", "-v", "ON_ERROR_STOP=1", "-c", query],
capture_output=True,
text=True,
env=os.environ.copy()
)
if result.returncode == 0:
print(result.stdout)
return result.stdout
else:
print(f"❌ Error: {result.stderr}")
return None
except Exception as e:
print(f"❌ Exception: {e}")
return None
# ============================================================================
# Q1: For session_id 789d3699-028e-4367-b515-b82e2cb5225f, what was the purchase price?
# ============================================================================
print("\n" + "-" * 80)
print("Q1: Purchase price for session_id 789d3699-028e-4367-b515-b82e2cb5225f")
print("-" * 80)
q1_sql = """
SELECT
session_id,
event_type,
price
FROM retail.events
WHERE session_id = '789d3699-028e-4367-b515-b82e2cb5225f'
AND event_type = 'purchase'
ORDER BY event_time;
"""
q1_result = run_psql_query(q1_sql, "Q1 SQL Query")
# Extract answer
q1_answer = None
if q1_result and "purchase" in q1_result.lower():
lines = q1_result.strip().split('\n')
for line in lines:
if '789d3699' in line:
parts = line.split('|')
if len(parts) >= 3:
q1_answer = parts[-1].strip()
break
print(f"\n📊 Q1 ANSWER: {q1_answer if q1_answer else 'Not found'}")
# ============================================================================
# Q2: How many products are sold by the brand "sokolov"?
# ============================================================================
print("\n" + "-" * 80)
print("Q2: How many products are sold by the brand 'sokolov'?")
print("-" * 80)
q2_sql = """
SELECT
COUNT(*) AS product_count
FROM retail.product p
WHERE p.brand = 'sokolov';
"""
q2_result = run_psql_query(q2_sql, "Q2 SQL Query")
print(f"\n📊 Q2 ANSWER: Extract from result above")
# ============================================================================
# Q3: What is the average purchase price of items purchased from the brand "febest"?
# ============================================================================
print("\n" + "-" * 80)
print("Q3: Average purchase price from brand 'febest'")
print("-" * 80)
q3_sql = """
SELECT
p.brand,
ROUND(AVG(e.price)::numeric, 2) AS avg_price
FROM retail.events e
JOIN retail.product p ON e.product_id = p.product_id
WHERE p.brand = 'febest'
AND e.event_type = 'purchase'
GROUP BY p.brand;
"""
q3_result = run_psql_query(q3_sql, "Q3 SQL Query")
print(f"\n📊 Q3 ANSWER: Extract from result above")
# ============================================================================
# Q4: What is average number of events per user?
# ============================================================================
print("\n" + "-" * 80)
print("Q4: Average number of events per user (2 decimal places)")
print("-" * 80)
q4_sql = """
SELECT
ROUND(
CAST(COUNT(*) AS numeric) / COUNT(DISTINCT s.user_id),
2
) AS avg_events_per_user
FROM retail.events e
JOIN retail.session s ON e.session_id = s.session_id;
"""
q4_result = run_psql_query(q4_sql, "Q4 SQL Query")
print(f"\n📊 Q4 ANSWER: Extract from result above (XX.XX format)")
# ============================================================================
# VERIFICATION: Count distinct users
# ============================================================================
print("\n" + "-" * 80)
print("VERIFICATION: Count distinct users")
print("-" * 80)
verify_sql = """
SELECT
COUNT(DISTINCT user_id) AS number_users
FROM retail.user;
"""
verify_result = run_psql_query(verify_sql, "Verification SQL Query")
print("\n✅ Expected answer: 3022290")
# ============================================================================
# SUMMARY
# ============================================================================
print("\n" + "=" * 80)
print("📋 ASSIGNMENT 2 SUMMARY")
print("=" * 80)
print("""
Q1: Purchase price for session 789d3699-028e-4367-b515-b82e2cb5225f
SQL: SELECT price FROM retail.events WHERE session_id = '...' AND event_type = 'purchase'
Answer: [Extract from result]
Q2: Products sold by brand 'sokolov'
SQL: SELECT COUNT(*) FROM retail.product WHERE brand = 'sokolov'
Answer: [Extract from result]
Q3: Average purchase price from brand 'febest'
SQL: SELECT AVG(e.price) FROM retail.events e
JOIN retail.product p ON e.product_id = p.product_id
WHERE p.brand = 'febest' AND e.event_type = 'purchase'
Answer: [Extract from result]
Q4: Average events per user
SQL: SELECT COUNT(*) / COUNT(DISTINCT user_id)
FROM retail.events e
JOIN retail.session s ON e.session_id = s.session_id
Answer: XX.XX
Verification: Total users = 3022290
""")
print("=" * 80)
================================================================================
🔍 ASSIGNMENT 2: QUERYING THE OPERATIONAL DATABASE
================================================================================
--------------------------------------------------------------------------------
Q1: Purchase price for session_id 789d3699-028e-4367-b515-b82e2cb5225f
--------------------------------------------------------------------------------
✅ Q1 SQL Query
❌ Error: psql: error: connection to server at "127.0.0.1", port 5433 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
📊 Q1 ANSWER: Not found
--------------------------------------------------------------------------------
Q2: How many products are sold by the brand 'sokolov'?
--------------------------------------------------------------------------------
✅ Q2 SQL Query
❌ Error: psql: error: connection to server at "127.0.0.1", port 5433 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
📊 Q2 ANSWER: Extract from result above
--------------------------------------------------------------------------------
Q3: Average purchase price from brand 'febest'
--------------------------------------------------------------------------------
✅ Q3 SQL Query
❌ Error: psql: error: connection to server at "127.0.0.1", port 5433 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
📊 Q3 ANSWER: Extract from result above
--------------------------------------------------------------------------------
Q4: Average number of events per user (2 decimal places)
--------------------------------------------------------------------------------
✅ Q4 SQL Query
❌ Error: psql: error: connection to server at "127.0.0.1", port 5433 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
📊 Q4 ANSWER: Extract from result above (XX.XX format)
--------------------------------------------------------------------------------
VERIFICATION: Count distinct users
--------------------------------------------------------------------------------
✅ Verification SQL Query
❌ Error: psql: error: connection to server at "127.0.0.1", port 5433 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
✅ Expected answer: 3022290
================================================================================
📋 ASSIGNMENT 2 SUMMARY
================================================================================
Q1: Purchase price for session 789d3699-028e-4367-b515-b82e2cb5225f
SQL: SELECT price FROM retail.events WHERE session_id = '...' AND event_type = 'purchase'
Answer: [Extract from result]
Q2: Products sold by brand 'sokolov'
SQL: SELECT COUNT(*) FROM retail.product WHERE brand = 'sokolov'
Answer: [Extract from result]
Q3: Average purchase price from brand 'febest'
SQL: SELECT AVG(e.price) FROM retail.events e
JOIN retail.product p ON e.product_id = p.product_id
WHERE p.brand = 'febest' AND e.event_type = 'purchase'
Answer: [Extract from result]
Q4: Average events per user
SQL: SELECT COUNT(*) / COUNT(DISTINCT user_id)
FROM retail.events e
JOIN retail.session s ON e.session_id = s.session_id
Answer: XX.XX
Verification: Total users = 3022290
================================================================================
In [10]:
# Cell 2: Define paths and load CSV files
BASE_DIR = "/home/bibawandaogo/data engineering 1/lab2_data"
# Vérifie que les fichiers existent
import os
csv_files = ["user.csv", "session.csv", "brand.csv", "category.csv",
"product.csv", "product_name.csv", "events.csv"]
print("✅ Checking CSV files:")
for csv_file in csv_files:
path = os.path.join(BASE_DIR, csv_file)
exists = os.path.exists(path)
size = os.path.getsize(path) if exists else 0
print(f" {csv_file}: {'✅' if exists else '❌'} ({size} bytes)")
print("\n✅ Loading DataFrames...")
# Charge tous les CSV
df_user = spark.read.option("header","true").option("inferSchema","true").csv(f"{BASE_DIR}/user.csv")
df_session = spark.read.option("header","true").option("inferSchema","true").csv(f"{BASE_DIR}/session.csv")
df_product = spark.read.option("header","true").option("inferSchema","true").csv(f"{BASE_DIR}/product.csv")
df_product_name = spark.read.option("header","true").option("inferSchema","true").csv(f"{BASE_DIR}/product_name.csv")
df_events = spark.read.option("header","true").option("inferSchema","true").csv(f"{BASE_DIR}/events.csv")
df_category = spark.read.option("header","true").option("inferSchema","true").csv(f"{BASE_DIR}/category.csv")
df_brand = spark.read.option("header","true").option("inferSchema","true").csv(f"{BASE_DIR}/brand.csv")
# Affiche les comptes
print("\n" + "=" * 60)
print("📊 Row Counts:")
print("=" * 60)
print(f"user: {df_user.count()}")
print(f"session: {df_session.count()}")
print(f"product: {df_product.count()}")
print(f"product_name: {df_product_name.count()}")
print(f"events: {df_events.count()}")
print(f"category: {df_category.count()}")
print(f"brand: {df_brand.count()}")
print("=" * 60)
✅ Checking CSV files: user.csv: ✅ (205 bytes) session.csv: ✅ (119 bytes) brand.csv: ✅ (151 bytes) category.csv: ✅ (165 bytes) product.csv: ✅ (346 bytes) product_name.csv: ✅ (411 bytes) events.csv: ✅ (914 bytes) ✅ Loading DataFrames... ============================================================ 📊 Row Counts: ============================================================ user: 10 session: 10 product: 10 product_name: 10 events: 20 category: 5 brand: 5 ============================================================
In [ ]:
# Cell: Querying Operational Database - Assignment 2
import subprocess
import pandas as pd
print("\n" + "=" * 80)
print("🔍 ASSIGNMENT 2: QUERYING THE OPERATIONAL DATABASE")
print("=" * 80)
def run_psql_query(query: str, description: str = "") -> str:
"""Execute psql query and return result"""
if description:
print(f"\n✅ {description}")
try:
result = subprocess.run(
["psql", "esiee_full", "-v", "ON_ERROR_STOP=1", "-c", query],
capture_output=True,
text=True,
env=os.environ.copy()
)
if result.returncode == 0:
print(result.stdout)
return result.stdout
else:
print(f"❌ Error: {result.stderr}")
return None
except Exception as e:
print(f"❌ Exception: {e}")
return None
# ============================================================================
# Q1: For session_id 789d3699-028e-4367-b515-b82e2cb5225f, what was the purchase price?
# ============================================================================
print("\n" + "-" * 80)
print("Q1: Purchase price for session_id 789d3699-028e-4367-b515-b82e2cb5225f")
print("-" * 80)
q1_sql = """
SELECT
session_id,
event_type,
price
FROM retail.events
WHERE session_id = '789d3699-028e-4367-b515-b82e2cb5225f'
AND event_type = 'purchase'
ORDER BY event_time;
"""
q1_result = run_psql_query(q1_sql, "Q1 SQL Query")
# Extract answer
q1_answer = None
if q1_result and "purchase" in q1_result.lower():
lines = q1_result.strip().split('\n')
for line in lines:
if '789d3699' in line:
parts = line.split('|')
if len(parts) >= 3:
q1_answer = parts[-1].strip()
break
print(f"\n📊 Q1 ANSWER: {q1_answer if q1_answer else 'Not found'}")
# ============================================================================
# Q2: How many products are sold by the brand "sokolov"?
# ============================================================================
print("\n" + "-" * 80)
print("Q2: How many products are sold by the brand 'sokolov'?")
print("-" * 80)
q2_sql = """
SELECT
COUNT(*) AS product_count
FROM retail.product p
WHERE p.brand = 'sokolov';
"""
q2_result = run_psql_query(q2_sql, "Q2 SQL Query")
print(f"\n📊 Q2 ANSWER: Extract from result above")
# ============================================================================
# Q3: What is the average purchase price of items purchased from the brand "febest"?
# ============================================================================
print("\n" + "-" * 80)
print("Q3: Average purchase price from brand 'febest'")
print("-" * 80)
q3_sql = """
SELECT
p.brand,
ROUND(AVG(e.price)::numeric, 2) AS avg_price
FROM retail.events e
JOIN retail.product p ON e.product_id = p.product_id
WHERE p.brand = 'febest'
AND e.event_type = 'purchase'
GROUP BY p.brand;
"""
q3_result = run_psql_query(q3_sql, "Q3 SQL Query")
print(f"\n📊 Q3 ANSWER: Extract from result above")
# ============================================================================
# Q4: What is average number of events per user?
# ============================================================================
print("\n" + "-" * 80)
print("Q4: Average number of events per user (2 decimal places)")
print("-" * 80)
q4_sql = """
SELECT
ROUND(
CAST(COUNT(*) AS numeric) / COUNT(DISTINCT s.user_id),
2
) AS avg_events_per_user
FROM retail.events e
JOIN retail.session s ON e.session_id = s.session_id;
"""
q4_result = run_psql_query(q4_sql, "Q4 SQL Query")
print(f"\n📊 Q4 ANSWER: Extract from result above (XX.XX format)")
# ============================================================================
# VERIFICATION: Count distinct users
# ============================================================================
print("\n" + "-" * 80)
print("VERIFICATION: Count distinct users")
print("-" * 80)
verify_sql = """
SELECT
COUNT(DISTINCT user_id) AS number_users
FROM retail.user;
"""
verify_result = run_psql_query(verify_sql, "Verification SQL Query")
print("\n✅ Expected answer: 3022290")
# ============================================================================
# SUMMARY
# ============================================================================
print("\n" + "=" * 80)
print("📋 ASSIGNMENT 2 SUMMARY")
print("=" * 80)
print("""
Q1: Purchase price for session 789d3699-028e-4367-b515-b82e2cb5225f
SQL: SELECT price FROM retail.events WHERE session_id = '...' AND event_type = 'purchase'
Answer: [Extract from result]
Q2: Products sold by brand 'sokolov'
SQL: SELECT COUNT(*) FROM retail.product WHERE brand = 'sokolov'
Answer: [Extract from result]
Q3: Average purchase price from brand 'febest'
SQL: SELECT AVG(e.price) FROM retail.events e
JOIN retail.product p ON e.product_id = p.product_id
WHERE p.brand = 'febest' AND e.event_type = 'purchase'
Answer: [Extract from result]
Q4: Average events per user
SQL: SELECT COUNT(*) / COUNT(DISTINCT user_id)
FROM retail.events e
JOIN retail.session s ON e.session_id = s.session_id
Answer: XX.XX
Verification: Total users = 3022290
""")
print("=" * 80)
================================================================================
🔍 ASSIGNMENT 2: QUERYING THE OPERATIONAL DATABASE
================================================================================
--------------------------------------------------------------------------------
Q1: Purchase price for session_id 789d3699-028e-4367-b515-b82e2cb5225f
--------------------------------------------------------------------------------
✅ Q1 SQL Query
❌ Error: psql: error: connection to server at "127.0.0.1", port 5433 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
📊 Q1 ANSWER: Not found
--------------------------------------------------------------------------------
Q2: How many products are sold by the brand 'sokolov'?
--------------------------------------------------------------------------------
✅ Q2 SQL Query
❌ Error: psql: error: connection to server at "127.0.0.1", port 5433 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
📊 Q2 ANSWER: Extract from result above
--------------------------------------------------------------------------------
Q3: Average purchase price from brand 'febest'
--------------------------------------------------------------------------------
✅ Q3 SQL Query
❌ Error: psql: error: connection to server at "127.0.0.1", port 5433 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
📊 Q3 ANSWER: Extract from result above
--------------------------------------------------------------------------------
Q4: Average number of events per user (2 decimal places)
--------------------------------------------------------------------------------
✅ Q4 SQL Query
❌ Error: psql: error: connection to server at "127.0.0.1", port 5433 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
📊 Q4 ANSWER: Extract from result above (XX.XX format)
--------------------------------------------------------------------------------
VERIFICATION: Count distinct users
--------------------------------------------------------------------------------
✅ Verification SQL Query
❌ Error: psql: error: connection to server at "127.0.0.1", port 5433 failed: Connection refused
Is the server running on that host and accepting TCP/IP connections?
✅ Expected answer: 3022290
================================================================================
📋 ASSIGNMENT 2 SUMMARY
================================================================================
Q1: Purchase price for session 789d3699-028e-4367-b515-b82e2cb5225f
SQL: SELECT price FROM retail.events WHERE session_id = '...' AND event_type = 'purchase'
Answer: [Extract from result]
Q2: Products sold by brand 'sokolov'
SQL: SELECT COUNT(*) FROM retail.product WHERE brand = 'sokolov'
Answer: [Extract from result]
Q3: Average purchase price from brand 'febest'
SQL: SELECT AVG(e.price) FROM retail.events e
JOIN retail.product p ON e.product_id = p.product_id
WHERE p.brand = 'febest' AND e.event_type = 'purchase'
Answer: [Extract from result]
Q4: Average events per user
SQL: SELECT COUNT(*) / COUNT(DISTINCT user_id)
FROM retail.events e
JOIN retail.session s ON e.session_id = s.session_id
Answer: XX.XX
Verification: Total users = 3022290
================================================================================
25/12/29 13:37:32 WARN HeartbeatReceiver: Removing executor driver with no recent heartbeats: 254579 ms exceeds timeout 120000 ms 25/12/29 13:37:33 WARN SparkContext: Killing executors is not supported by current scheduler. 25/12/29 13:37:35 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:37:35 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:37:45 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:37:45 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:37:55 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:37:55 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:38:05 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:38:05 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:38:15 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:38:15 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:38:25 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:38:25 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:38:36 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:38:36 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:38:46 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:38:46 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:38:56 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:38:56 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:39:06 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:39:06 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:39:16 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:39:16 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:39:26 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:39:26 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:39:36 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:39:36 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:39:46 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:39:46 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:39:57 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:39:57 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:40:07 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:40:07 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:40:17 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:40:17 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:40:27 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:40:27 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:40:37 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:40:37 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:40:47 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:40:47 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:41:30 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:41:30 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:41:40 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:41:40 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:41:50 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:41:50 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:42:01 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:42:01 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:42:11 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:42:11 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:42:21 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:42:21 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:42:31 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:42:31 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:42:41 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:42:41 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:42:51 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:42:51 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:43:01 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:43:01 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:43:11 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:43:11 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:43:22 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:43:22 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:43:32 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:43:32 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:43:42 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:43:42 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:43:52 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:43:52 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:44:02 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:44:02 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:44:12 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:44:12 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:44:22 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.dispatchOrAddCallbacks(Promise.scala:312) at scala.concurrent.impl.Promise$DefaultPromise.flatMap(Promise.scala:176) at org.apache.spark.rpc.netty.NettyRpcEnv.asyncSetupEndpointRefByURI(NettyRpcEnv.scala:153) ... 17 more 25/12/29 13:44:22 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.dispatchOrAddCallbacks(Promise.scala:312) at scala.concurrent.impl.Promise$DefaultPromise.flatMap(Promise.scala:176) at org.apache.spark.rpc.netty.NettyRpcEnv.asyncSetupEndpointRefByURI(NettyRpcEnv.scala:153) ... 17 more 25/12/29 13:44:33 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:44:33 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:44:43 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:44:43 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:44:53 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:44:53 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:45:03 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:45:03 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:45:13 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:45:13 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:45:23 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:45:23 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:50:51 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:50:51 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:51:01 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:51:01 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:51:11 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:51:11 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:51:22 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:51:22 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:51:32 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:51:32 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:51:42 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:51:42 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:51:52 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:51:52 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:52:02 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:52:02 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:52:12 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:52:12 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:52:23 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:52:23 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:52:33 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:52:33 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:52:43 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:52:43 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:52:53 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.dispatchOrAddCallbacks(Promise.scala:312) at scala.concurrent.impl.Promise$DefaultPromise.flatMap(Promise.scala:176) at org.apache.spark.rpc.netty.NettyRpcEnv.asyncSetupEndpointRefByURI(NettyRpcEnv.scala:153) ... 17 more 25/12/29 13:52:53 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.dispatchOrAddCallbacks(Promise.scala:312) at scala.concurrent.impl.Promise$DefaultPromise.flatMap(Promise.scala:176) at org.apache.spark.rpc.netty.NettyRpcEnv.asyncSetupEndpointRefByURI(NettyRpcEnv.scala:153) ... 17 more 25/12/29 13:53:03 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:53:03 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:53:13 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:53:13 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:53:23 WARN Executor: Issue communicating with driver in heartbeater org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:101) at org.apache.spark.rpc.RpcEndpointRef.askSync(RpcEndpointRef.scala:85) at org.apache.spark.storage.BlockManagerMaster.registerBlockManager(BlockManagerMaster.scala:81) at org.apache.spark.storage.BlockManager.reregister(BlockManager.scala:669) at org.apache.spark.executor.Executor.reportHeartBeat(Executor.scala:1296) at org.apache.spark.executor.Executor.$anonfun$heartbeater$1(Executor.scala:307) at scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.scala:18) at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1937) at org.apache.spark.Heartbeater$$anon$1.run(Heartbeater.scala:46) at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572) at java.base/java.util.concurrent.FutureTask.runAndReset(FutureTask.java:358) at java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:305) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) ... 3 more Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:53:23 ERROR Inbox: Ignoring error org.apache.spark.SparkException: Exception thrown in awaitResult: at org.apache.spark.util.SparkThreadUtils$.awaitResult(SparkThreadUtils.scala:53) at org.apache.spark.util.ThreadUtils$.awaitResult(ThreadUtils.scala:342) at org.apache.spark.rpc.RpcTimeout.awaitResult(RpcTimeout.scala:75) at org.apache.spark.rpc.RpcEnv.setupEndpointRefByURI(RpcEnv.scala:102) at org.apache.spark.rpc.RpcEnv.setupEndpointRef(RpcEnv.scala:110) at org.apache.spark.util.RpcUtils$.makeDriverRef(RpcUtils.scala:36) at org.apache.spark.storage.BlockManagerMasterEndpoint.driverEndpoint$lzycompute(BlockManagerMasterEndpoint.scala:132) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$driverEndpoint(BlockManagerMasterEndpoint.scala:131) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$lzycompute$1(BlockManagerMasterEndpoint.scala:700) at org.apache.spark.storage.BlockManagerMasterEndpoint.isExecutorAlive$1(BlockManagerMasterEndpoint.scala:699) at org.apache.spark.storage.BlockManagerMasterEndpoint.org$apache$spark$storage$BlockManagerMasterEndpoint$$register(BlockManagerMasterEndpoint.scala:739) at org.apache.spark.storage.BlockManagerMasterEndpoint$$anonfun$receiveAndReply$1.applyOrElse(BlockManagerMasterEndpoint.scala:141) at org.apache.spark.rpc.netty.Inbox.$anonfun$process$1(Inbox.scala:104) at org.apache.spark.rpc.netty.Inbox.safelyCall(Inbox.scala:216) at org.apache.spark.rpc.netty.Inbox.process(Inbox.scala:101) at org.apache.spark.rpc.netty.MessageLoop.org$apache$spark$rpc$netty$MessageLoop$$receiveLoop(MessageLoop.scala:76) at org.apache.spark.rpc.netty.MessageLoop$$anon$1.run(MessageLoop.scala:42) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642) at java.base/java.lang.Thread.run(Thread.java:1583) Caused by: org.apache.spark.rpc.RpcEndpointNotFoundException: Cannot find endpoint: spark://CoarseGrainedScheduler@10.255.255.254:44675 at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1(NettyRpcEnv.scala:151) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$asyncSetupEndpointRefByURI$1$adapted(NettyRpcEnv.scala:147) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:470) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:504) at scala.concurrent.ExecutionContext$parasitic$.execute(ExecutionContext.scala:222) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:335) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.trySuccess(Promise.scala:99) at scala.concurrent.Promise.trySuccess$(Promise.scala:99) at scala.concurrent.impl.Promise$DefaultPromise.trySuccess(Promise.scala:104) at org.apache.spark.rpc.netty.NettyRpcEnv.onSuccess$1(NettyRpcEnv.scala:228) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5(NettyRpcEnv.scala:242) at org.apache.spark.rpc.netty.NettyRpcEnv.$anonfun$askAbortable$5$adapted(NettyRpcEnv.scala:241) at scala.concurrent.impl.Promise$Transformation.run(Promise.scala:484) at org.apache.spark.util.ThreadUtils$$anon$1.execute(ThreadUtils.scala:99) at scala.concurrent.impl.ExecutionContextImpl.execute(ExecutionContextImpl.scala:21) at scala.concurrent.impl.Promise$Transformation.submitWithValue(Promise.scala:429) at scala.concurrent.impl.Promise$DefaultPromise.submitWithValue(Promise.scala:338) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete0(Promise.scala:285) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:278) at scala.concurrent.Promise.complete(Promise.scala:57) at scala.concurrent.Promise.complete$(Promise.scala:56) at scala.concurrent.impl.Promise$DefaultPromise.complete(Promise.scala:104) at scala.concurrent.Promise.success(Promise.scala:91) at scala.concurrent.Promise.success$(Promise.scala:91) at scala.concurrent.impl.Promise$DefaultPromise.success(Promise.scala:104) at org.apache.spark.rpc.netty.LocalNettyRpcCallContext.send(NettyRpcCallContext.scala:50) at org.apache.spark.rpc.netty.NettyRpcCallContext.reply(NettyRpcCallContext.scala:32) at org.apache.spark.rpc.netty.RpcEndpointVerifier$$anonfun$receiveAndReply$1.applyOrElse(RpcEndpointVerifier.scala:31) ... 8 more 25/12/29 13:53:23 ERROR Executor: Exit as unable to send heartbeats to driver more than 60 times
In [4]:
# Cell 3: Build dim_user
print("✅ Building dim_user...")
# Ajoute la génération basée sur l'année de naissance
dim_user = (
df_user
.withColumn("birthdate", F.to_date(col("birthdate")))
.withColumn("birth_year", F.year(col("birthdate")))
.withColumn("generation",
F.when((col("birth_year") >= 1925) & (col("birth_year") <= 1945), "Traditionalists")
.when((col("birth_year") >= 1946) & (col("birth_year") <= 1964), "Boomers")
.when((col("birth_year") >= 1965) & (col("birth_year") <= 1980), "GenX")
.when((col("birth_year") >= 1981) & (col("birth_year") <= 2000), "Millennials")
.when((col("birth_year") >= 2001) & (col("birth_year") <= 2020), "GenZ")
.otherwise("Unknown")
)
.withColumn("user_key", F.dense_rank().over(Window.orderBy(col("user_id"))))
.select("user_key", "user_id", "gender", "birthdate", "generation")
)
print(f"✅ dim_user created with {dim_user.count()} rows")
dim_user.show(5)
✅ Building dim_user... ✅ dim_user created with 10 rows
25/12/08 23:13:12 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:13:12 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:13:12 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
+--------+-------+------+----------+-----------+ |user_key|user_id|gender| birthdate| generation| +--------+-------+------+----------+-----------+ | 1| U001| M|1980-05-15| GenX| | 2| U002| F|1995-08-22|Millennials| | 3| U003| M|1975-12-03| GenX| | 4| U004| F|1990-03-17|Millennials| | 5| U005| M|1985-07-09|Millennials| +--------+-------+------+----------+-----------+ only showing top 5 rows
In [5]:
# Cell 4: Build dim_age
print("✅ Building dim_age...")
age_band_rows = [
("<18", None, 17),
("18-24", 18, 24),
("25-34", 25, 34),
("35-44", 35, 44),
("45-54", 45, 54),
("55-64", 55, 64),
("65-74", 65, 74),
("75-84", 75, 84),
("85-94", 85, 94),
("unknown", None, None),
]
dim_age = spark.createDataFrame(age_band_rows, ["age_band", "min_age", "max_age"])
w_age = Window.orderBy(F.col("age_band"))
dim_age = dim_age.withColumn("age_key", F.dense_rank().over(w_age))
dim_age = dim_age.select("age_key", "age_band", "min_age", "max_age")
print(f"✅ dim_age created with {dim_age.count()} rows")
dim_age.show()
✅ Building dim_age...
25/12/08 23:13:33 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:13:33 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:13:33 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
✅ dim_age created with 10 rows +-------+--------+-------+-------+ |age_key|age_band|min_age|max_age| +-------+--------+-------+-------+ | 1| 18-24| 18| 24| | 2| 25-34| 25| 34| | 3| 35-44| 35| 44| | 4| 45-54| 45| 54| | 5| 55-64| 55| 64| | 6| 65-74| 65| 74| | 7| 75-84| 75| 84| | 8| 85-94| 85| 94| | 9| <18| NULL| 17| | 10| unknown| NULL| NULL| +-------+--------+-------+-------+
In [6]:
# Cell 5: Build dim_brand
print("✅ Building dim_brand...")
dim_brand = (
df_brand
.withColumn("brand_key", F.dense_rank().over(Window.orderBy(col("brand"))))
.select("brand_key", F.col("brand").alias("brand_code"), F.col("description").alias("brand_desc"))
)
print(f"✅ dim_brand created with {dim_brand.count()} rows")
dim_brand.show()
✅ Building dim_brand... ✅ dim_brand created with 5 rows +---------+----------+--------------------+ |brand_key|brand_code| brand_desc| +---------+----------+--------------------+ | 1| Brand_A| Premium electronics| | 2| Brand_B|Budget household ...| | 3| Brand_C| Sports equipment| | 4| Brand_D| Fashion apparel| | 5| Brand_E| Home furnishings| +---------+----------+--------------------+
25/12/08 23:15:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
In [7]:
# Cell 6: Build dim_category
print("✅ Building dim_category...")
dim_category = (
df_category
.withColumn("category_key", F.dense_rank().over(Window.orderBy(col("category"))))
.select("category_key", F.col("category").alias("category_code"), F.col("description").alias("category_desc"))
)
print(f"✅ dim_category created with {dim_category.count()} rows")
dim_category.show()
✅ Building dim_category... ✅ dim_category created with 5 rows +------------+-------------+--------------------+ |category_key|category_code| category_desc| +------------+-------------+--------------------+ | 1| Electronics| Electronic devices| | 2| Fashion|Clothing and acce...| | 3| Furniture| Home furnishings| | 4| Household| Home and kitchen| | 5| Sports| Sports and outdoor| +------------+-------------+--------------------+
25/12/08 23:15:25 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:25 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:25 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
In [8]:
# Cell 7: Build dim_product
print("✅ Building dim_product...")
# Join product with product_name pour ajouter les descriptions
df_product_enriched = (
df_product
.join(df_product_name, on=["category", "product_name"], how="left")
.select("product_id", "brand", "category", "product_name", "description")
)
# Join avec dim_brand pour obtenir brand_key
df_product_with_brand = (
df_product_enriched
.join(dim_brand.select("brand_key", "brand_code"),
df_product_enriched.brand == dim_brand.brand_code,
how="left")
.select(
F.col("product_id"),
F.col("product_name").alias("product_desc"),
F.col("brand_key")
)
)
# Join avec dim_category pour obtenir category_key
dim_product = (
df_product_enriched
.join(dim_category.select("category_key", "category_code"),
df_product_enriched.category == dim_category.category_code,
how="left")
.join(dim_brand.select("brand_key", "brand_code"),
df_product_enriched.brand == dim_brand.brand_code,
how="left")
.withColumn("product_key", F.dense_rank().over(Window.orderBy(col("product_id"))))
.select("product_key", "product_id", F.col("description").alias("product_desc"), "brand_key", "category_key")
)
print(f"✅ dim_product created with {dim_product.count()} rows")
dim_product.show()
✅ Building dim_product... ✅ dim_product created with 10 rows
25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
+-----------+----------+--------------------+---------+------------+ |product_key|product_id| product_desc|brand_key|category_key| +-----------+----------+--------------------+---------+------------+ | 1| P001|Portable computer...| 1| 1| | 2| P002|Audio device for ...| 1| 1| | 3| P003|Kitchen appliance...| 2| 4| | 4| P004|Bread toasting de...| 2| 4| | 5| P005| Athletic footwear| 3| 5| | 6| P006| Exercise surface| 3| 5| | 7| P007| Casual clothing| 4| 2| | 8| P008| Denim pants| 4| 2| | 9| P009| Seating furniture| 5| 3| | 10| P010| Dining surface| 5| 3| +-----------+----------+--------------------+---------+------------+
25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:15:40 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
In [9]:
# Cell 8: Build dim_date
print("✅ Building dim_date...")
# Convertis event_time en date
df_events_with_date = df_events.withColumn("event_date", F.to_date(F.col("event_time")))
# Trouve la plage de dates
min_date = df_events_with_date.agg(F.min("event_date")).collect()[0][0]
max_date = df_events_with_date.agg(F.max("event_date")).collect()[0][0]
print(f"Date range: {min_date} to {max_date}")
# Génère toutes les dates dans la plage
dim_date = (
spark.sql(f"SELECT explode(sequence(to_date('{min_date}'), to_date('{max_date}'))) as date")
.withColumn("year", F.year(F.col("date")))
.withColumn("month", F.month(F.col("date")))
.withColumn("day", F.dayofmonth(F.col("date")))
.withColumn("day_of_week", F.dayofweek(F.col("date")))
.withColumn("day_name", F.date_format(F.col("date"), "EEEE"))
.withColumn("is_weekend", (F.col("day_of_week") == 1) | (F.col("day_of_week") == 7))
.withColumn("week_of_year", F.weekofyear(F.col("date")))
.withColumn("month_name", F.date_format(F.col("date"), "MMMM"))
.withColumn("quarter", F.quarter(F.col("date")))
.withColumn("date_key", F.col("year") * 10000 + F.col("month") * 100 + F.col("day"))
.select("date_key", "date", "day", "day_of_week", "day_name", "is_weekend",
"week_of_year", "month", "month_name", "quarter", "year")
)
print(f"✅ dim_date created with {dim_date.count()} rows")
dim_date.show()
✅ Building dim_date... Date range: 2024-12-01 to 2024-12-04 ✅ dim_date created with 4 rows +--------+----------+---+-----------+---------+----------+------------+-----+----------+-------+----+ |date_key| date|day|day_of_week| day_name|is_weekend|week_of_year|month|month_name|quarter|year| +--------+----------+---+-----------+---------+----------+------------+-----+----------+-------+----+ |20241201|2024-12-01| 1| 1| Sunday| true| 48| 12| December| 4|2024| |20241202|2024-12-02| 2| 2| Monday| false| 49| 12| December| 4|2024| |20241203|2024-12-03| 3| 3| Tuesday| false| 49| 12| December| 4|2024| |20241204|2024-12-04| 4| 4|Wednesday| false| 49| 12| December| 4|2024| +--------+----------+---+-----------+---------+----------+------------+-----+----------+-------+----+
In [10]:
# Cell 9: Summary of all dimensions
print("\n" + "=" * 60)
print("📊 DIMENSION TABLES SUMMARY")
print("=" * 60)
print(f"dim_user: {dim_user.count()} rows")
print(f"dim_age: {dim_age.count()} rows")
print(f"dim_brand: {dim_brand.count()} rows")
print(f"dim_category: {dim_category.count()} rows")
print(f"dim_product: {dim_product.count()} rows")
print(f"dim_date: {dim_date.count()} rows")
print("=" * 60)
# Affiche les schémas
print("\n✅ Schemas:")
print("\ndim_user:")
dim_user.printSchema()
print("\ndim_product:")
dim_product.printSchema()
print("\ndim_date:")
dim_date.printSchema()
============================================================ 📊 DIMENSION TABLES SUMMARY ============================================================ dim_user: 10 rows dim_age: 10 rows dim_brand: 5 rows dim_category: 5 rows dim_product: 10 rows dim_date: 4 rows ============================================================ ✅ Schemas: dim_user: root |-- user_key: integer (nullable = false) |-- user_id: string (nullable = true) |-- gender: string (nullable = true) |-- birthdate: date (nullable = true) |-- generation: string (nullable = false) dim_product: root |-- product_key: integer (nullable = false) |-- product_id: string (nullable = true) |-- product_desc: string (nullable = true) |-- brand_key: integer (nullable = true) |-- category_key: integer (nullable = true) dim_date: root |-- date_key: integer (nullable = false) |-- date: date (nullable = false) |-- day: integer (nullable = false) |-- day_of_week: integer (nullable = false) |-- day_name: string (nullable = false) |-- is_weekend: boolean (nullable = false) |-- week_of_year: integer (nullable = false) |-- month: integer (nullable = false) |-- month_name: string (nullable = false) |-- quarter: integer (nullable = false) |-- year: integer (nullable = false)
In [11]:
# Cell 10: Clean Events
print("✅ Cleaning events...")
# Convertis event_time en timestamp
df_events_clean = (
df_events
.withColumn("event_time", F.to_timestamp(col("event_time")))
.withColumn("event_date", F.to_date(F.col("event_time")))
.withColumn("price", F.col("price").cast("double"))
)
# Filtre les événements invalides
valid_types = ["view", "cart", "purchase", "remove"]
events_clean = (
df_events_clean
.filter(F.col("event_time").isNotNull())
.filter(F.col("session_id").isNotNull())
.filter(F.col("product_id").isNotNull())
.filter((F.col("price").isNull()) | (F.col("price") >= 0))
.filter(F.col("event_type").isin(valid_types))
.filter(F.col("event_date") <= F.current_date())
)
print(f"✅ events_clean: {events_clean.count()} rows")
events_clean.show(5)
✅ Cleaning events... ✅ events_clean: 20 rows +-------------------+----------+----------+----------+------+----------+ | event_time|event_type|session_id|product_id| price|event_date| +-------------------+----------+----------+----------+------+----------+ |2024-12-01 10:30:00| view| S001| P001|1200.0|2024-12-01| |2024-12-01 10:35:00| cart| S001| P001|1200.0|2024-12-01| |2024-12-01 10:40:00| purchase| S001| P001|1200.0|2024-12-01| |2024-12-01 11:00:00| view| S002| P003| 89.99|2024-12-01| |2024-12-01 11:05:00| purchase| S002| P003| 89.99|2024-12-01| +-------------------+----------+----------+----------+------+----------+ only showing top 5 rows
In [12]:
# Cell 11: Analyze prices
print("✅ Price Statistics...")
price_stats = events_clean.agg(
F.min("price").alias("minimum"),
F.max("price").alias("maximum"),
F.avg("price").alias("average"),
F.count("price").alias("count_non_null")
).collect()[0]
minimum = price_stats["minimum"]
maximum = price_stats["maximum"]
average = price_stats["average"]
print(f"Minimum price: {minimum}")
print(f"Maximum price: {maximum}")
print(f"Average price: {average:.2f}")
print(f"Non-null prices: {price_stats['count_non_null']}")
# Calcule le threshold: 100x la moyenne
threshold = (average or 0) * 100
print(f"\n🔍 Price threshold (100x average): {threshold:.2f}")
# Filtre les prix excessifs
events_clean = events_clean.filter(
(F.col("price").isNull()) | (F.col("price") <= threshold)
)
print(f"✅ After filtering expensive items: {events_clean.count()} rows")
✅ Price Statistics... Minimum price: 25.0 Maximum price: 1200.0 Average price: 290.24 Non-null prices: 20 🔍 Price threshold (100x average): 29024.45 ✅ After filtering expensive items: 20 rows
In [13]:
# Cell 12: Create lookup tables
print("✅ Creating lookup tables...")
# Lookup: user_id → user_key
user_lkp = dim_user.select("user_id", "user_key")
# Lookup: product_id → product_key, brand_key, category_key
prod_lkp = dim_product.select("product_id", "product_key", "brand_key", "category_key")
# Lookup: date → date_key
date_lkp = dim_date.select("date", "date_key")
# Bridge: session_id → user_id
session_bridge = df_session.select("session_id", "user_id")
print(f"user_lkp: {user_lkp.count()}")
print(f"prod_lkp: {prod_lkp.count()}")
print(f"date_lkp: {date_lkp.count()}")
print(f"session_bridge: {session_bridge.count()}")
✅ Creating lookup tables... user_lkp: 10 prod_lkp: 10 date_lkp: 4 session_bridge: 10
In [14]:
# Cell 13: Build fact_events
print("✅ Building fact_events...")
# Démarre avec les événements nettoyés
fact_events = events_clean.select(
"event_time", "event_type", "session_id", "product_id", "price", "event_date"
)
# Join 1: Récupère user_id via session_id
fact_events = (
fact_events
.join(session_bridge, on="session_id", how="left")
)
# Join 2: Récupère product_key, brand_key, category_key
fact_events = (
fact_events
.join(prod_lkp, on="product_id", how="left")
)
# Join 3: Récupère date_key
fact_events = (
fact_events
.join(date_lkp, fact_events.event_date == date_lkp.date, how="left")
.drop("date")
)
# Join 4: Récupère user_key et birthdate
fact_events = (
fact_events
.join(user_lkp, on="user_id", how="left")
.join(dim_user.select("user_key", "birthdate"), on="user_key", how="left")
)
# Calcule l'âge au moment de l'événement
fact_events = fact_events.withColumn(
"age_on_event",
F.floor(F.months_between(F.col("event_date"), F.to_date("birthdate"))/12)
)
# Join 5: Récupère age_key basé sur age_on_event
fact_events = (
fact_events
.join(
dim_age.select("age_key", "age_band", "min_age", "max_age"),
(
((F.col("age_on_event") > F.col("min_age"))) &
((F.col("age_on_event") <= F.col("max_age")))
),
"left"
)
)
# Sélectionne les colonnes finales
fact_events = fact_events.select(
"date_key",
"user_key",
"age_key",
"product_key",
"brand_key",
"category_key",
"session_id",
"event_time",
"event_type",
"price"
)
print(f"✅ fact_events created with {fact_events.count()} rows")
fact_events.show(10)
✅ Building fact_events...
25/12/08 23:18:00 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:00 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:00 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:00 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:00 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:00 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:00 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:00 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:00 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:00 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
✅ fact_events created with 20 rows
25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:18:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
+--------+--------+-------+-----------+---------+------------+----------+-------------------+----------+------+ |date_key|user_key|age_key|product_key|brand_key|category_key|session_id| event_time|event_type| price| +--------+--------+-------+-----------+---------+------------+----------+-------------------+----------+------+ |20241202| 3| 4| 7| 4| 2| S004|2024-12-02 14:30:00| view| 29.99| |20241202| 3| 4| 7| 4| 2| S004|2024-12-02 14:35:00| purchase| 29.99| |20241201| 2| 2| 3| 2| 4| S002|2024-12-01 11:00:00| view| 89.99| |20241201| 2| 2| 3| 2| 4| S002|2024-12-01 11:05:00| purchase| 89.99| |20241201| 1| 3| 1| 1| 1| S001|2024-12-01 10:30:00| view|1200.0| |20241201| 1| 3| 1| 1| 1| S001|2024-12-01 10:35:00| cart|1200.0| |20241201| 1| 3| 1| 1| 1| S001|2024-12-01 10:40:00| purchase|1200.0| |20241203| 4| 2| 9| 5| 3| S005|2024-12-03 08:00:00| view|299.99| |20241202| 1| 3| 5| 3| 5| S003|2024-12-02 09:00:00| view| 150.0| |20241202| 1| 3| 5| 3| 5| S003|2024-12-02 09:15:00| cart| 150.0| +--------+--------+-------+-----------+---------+------------+----------+-------------------+----------+------+ only showing top 10 rows
In [15]:
# Cell 14: Display fact_events details
print("\n" + "=" * 70)
print("📊 FACT_EVENTS TABLE")
print("=" * 70)
fact_events.printSchema()
print(f"\nTotal rows: {fact_events.count()}")
print("\nSample data:")
fact_events.show(10, truncate=False)
print("\n" + "=" * 70)
print("✅ STAR SCHEMA COMPLETE!")
print("=" * 70)
====================================================================== 📊 FACT_EVENTS TABLE ====================================================================== root |-- date_key: integer (nullable = true) |-- user_key: integer (nullable = true) |-- age_key: integer (nullable = true) |-- product_key: integer (nullable = true) |-- brand_key: integer (nullable = true) |-- category_key: integer (nullable = true) |-- session_id: string (nullable = true) |-- event_time: timestamp (nullable = true) |-- event_type: string (nullable = true) |-- price: double (nullable = true)
25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:35 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
Total rows: 20 Sample data:
25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:36 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:37 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:37 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:37 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:37 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:37 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:37 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:37 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:20:37 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
+--------+--------+-------+-----------+---------+------------+----------+-------------------+----------+------+ |date_key|user_key|age_key|product_key|brand_key|category_key|session_id|event_time |event_type|price | +--------+--------+-------+-----------+---------+------------+----------+-------------------+----------+------+ |20241202|3 |4 |7 |4 |2 |S004 |2024-12-02 14:30:00|view |29.99 | |20241202|3 |4 |7 |4 |2 |S004 |2024-12-02 14:35:00|purchase |29.99 | |20241201|2 |2 |3 |2 |4 |S002 |2024-12-01 11:00:00|view |89.99 | |20241201|2 |2 |3 |2 |4 |S002 |2024-12-01 11:05:00|purchase |89.99 | |20241201|1 |3 |1 |1 |1 |S001 |2024-12-01 10:30:00|view |1200.0| |20241201|1 |3 |1 |1 |1 |S001 |2024-12-01 10:35:00|cart |1200.0| |20241201|1 |3 |1 |1 |1 |S001 |2024-12-01 10:40:00|purchase |1200.0| |20241203|4 |2 |9 |5 |3 |S005 |2024-12-03 08:00:00|view |299.99| |20241202|1 |3 |5 |3 |5 |S003 |2024-12-02 09:00:00|view |150.0 | |20241202|1 |3 |5 |3 |5 |S003 |2024-12-02 09:15:00|cart |150.0 | +--------+--------+-------+-----------+---------+------------+----------+-------------------+----------+------+ only showing top 10 rows ====================================================================== ✅ STAR SCHEMA COMPLETE! ======================================================================
In [16]:
# Cell 15: Quality Gates
print("\n" + "=" * 70)
print("🔍 QUALITY GATES")
print("=" * 70)
# Gate 1: Verify row count is non-zero
gate_1_count = fact_events.count()
gate_1_pass = gate_1_count > 0
print(f"\n✅ GATE 1: Row count non-zero")
print(f" Rows: {gate_1_count}")
print(f" Status: {'✅ PASS' if gate_1_pass else '❌ FAIL'}")
if not gate_1_pass:
raise Exception("GATE 1 FAILED: No rows in fact_events!")
# Gate 2: Check null rate thresholds
print(f"\n✅ GATE 2: Null rate thresholds")
null_checks = {
"date_key": 0.05, # Max 5% nulls
"user_key": 0.05, # Max 5% nulls
"product_key": 0.05, # Max 5% nulls
"event_type": 0.01, # Max 1% nulls
"price": 0.20, # Max 20% nulls (views don't have prices)
}
gate_2_pass = True
for col_name, threshold in null_checks.items():
null_count = fact_events.filter(F.col(col_name).isNull()).count()
null_rate = null_count / gate_1_count
passed = null_rate <= threshold
gate_2_pass = gate_2_pass and passed
status = "✅" if passed else "❌"
print(f" {status} {col_name}: {null_rate:.2%} (threshold: {threshold:.2%})")
print(f" Status: {'✅ PASS' if gate_2_pass else '❌ FAIL'}")
if not gate_2_pass:
raise Exception("GATE 2 FAILED: Null rate threshold exceeded!")
# Gate 3: Referential integrity checks (FK coverage)
print(f"\n✅ GATE 3: Referential integrity (FK coverage)")
# Check date_key references
date_keys_in_fact = set(fact_events.select("date_key").rdd.flatMap(lambda x: x).collect())
date_keys_in_dim = set(dim_date.select("date_key").rdd.flatMap(lambda x: x).collect())
missing_dates = date_keys_in_fact - date_keys_in_dim
# Check user_key references
user_keys_in_fact = set(fact_events.filter(F.col("user_key").isNotNull()).select("user_key").rdd.flatMap(lambda x: x).collect())
user_keys_in_dim = set(dim_user.select("user_key").rdd.flatMap(lambda x: x).collect())
missing_users = user_keys_in_fact - user_keys_in_dim
# Check product_key references
product_keys_in_fact = set(fact_events.filter(F.col("product_key").isNotNull()).select("product_key").rdd.flatMap(lambda x: x).collect())
product_keys_in_dim = set(dim_product.select("product_key").rdd.flatMap(lambda x: x).collect())
missing_products = product_keys_in_fact - product_keys_in_dim
gate_3_pass = (len(missing_dates) == 0) and (len(missing_users) == 0) and (len(missing_products) == 0)
print(f" Date references: {len(missing_dates)} missing")
print(f" User references: {len(missing_users)} missing")
print(f" Product references: {len(missing_products)} missing")
print(f" Status: {'✅ PASS' if gate_3_pass else '❌ FAIL'}")
if not gate_3_pass:
raise Exception("GATE 3 FAILED: Referential integrity broken!")
# Final verdict
print("\n" + "=" * 70)
all_gates_pass = gate_1_pass and gate_2_pass and gate_3_pass
if all_gates_pass:
print("✅ ALL QUALITY GATES PASSED!")
else:
print("❌ SOME GATES FAILED - CHECK ABOVE")
print("=" * 70)
====================================================================== 🔍 QUALITY GATES ======================================================================
25/12/08 23:21:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:01 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
✅ GATE 1: Row count non-zero Rows: 20 Status: ✅ PASS ✅ GATE 2: Null rate thresholds
25/12/08 23:21:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:02 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
✅ date_key: 0.00% (threshold: 5.00%)
25/12/08 23:21:03 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:03 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:04 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:04 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:04 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:04 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:04 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:04 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
✅ user_key: 0.00% (threshold: 5.00%)
25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
✅ product_key: 0.00% (threshold: 5.00%)
25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:05 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
✅ event_type: 0.00% (threshold: 1.00%)
25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:06 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
✅ price: 0.00% (threshold: 20.00%) Status: ✅ PASS ✅ GATE 3: Referential integrity (FK coverage)
25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:07 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:08 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:09 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:10 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:10 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:10 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:10 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:10 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:10 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:10 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:10 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:10 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:10 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:10 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
Date references: 0 missing User references: 0 missing Product references: 0 missing Status: ✅ PASS ====================================================================== ✅ ALL QUALITY GATES PASSED! ======================================================================
In [17]:
# Cell 16: Export to CSV and Parquet
import os
import shutil
OUTPUT_DIR = "/home/bibawandaogo/data engineering 1/lab2_output"
# Crée le répertoire de sortie
os.makedirs(OUTPUT_DIR, exist_ok=True)
print("\n" + "=" * 70)
print("📤 EXPORTING OUTPUTS")
print("=" * 70)
# 1. CSV (uncompressed)
print("\n✅ Writing CSV (no compression)...")
csv_uncompressed = f"{OUTPUT_DIR}/fact_events_csv"
if os.path.exists(csv_uncompressed):
shutil.rmtree(csv_uncompressed)
fact_events.coalesce(1).write.mode("overwrite").option("header", "true").csv(csv_uncompressed)
print(f" ✅ Saved to {csv_uncompressed}")
# 2. CSV (Snappy compressed)
print("\n✅ Writing CSV (Snappy compressed)...")
csv_snappy = f"{OUTPUT_DIR}/fact_events_csv_snappy"
if os.path.exists(csv_snappy):
shutil.rmtree(csv_snappy)
fact_events.coalesce(1).write.mode("overwrite").option("header", "true").option("compression", "snappy").csv(csv_snappy)
print(f" ✅ Saved to {csv_snappy}")
# 3. Parquet (default compression)
print("\n✅ Writing Parquet...")
parquet_path = f"{OUTPUT_DIR}/fact_events_parquet"
if os.path.exists(parquet_path):
shutil.rmtree(parquet_path)
fact_events.coalesce(1).write.mode("overwrite").parquet(parquet_path)
print(f" ✅ Saved to {parquet_path}")
print("\n" + "=" * 70)
====================================================================== 📤 EXPORTING OUTPUTS ====================================================================== ✅ Writing CSV (no compression)...
25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:49 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
✅ Saved to /home/bibawandaogo/data engineering 1/lab2_output/fact_events_csv ✅ Writing CSV (Snappy compressed)...
25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:50 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
✅ Saved to /home/bibawandaogo/data engineering 1/lab2_output/fact_events_csv_snappy ✅ Writing Parquet...
25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:21:51 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
✅ Saved to /home/bibawandaogo/data engineering 1/lab2_output/fact_events_parquet ======================================================================
In [18]:
# Cell 17: Compare file sizes
import os
print("\n" + "=" * 70)
print("📊 FILE SIZE COMPARISON")
print("=" * 70)
def get_dir_size(path):
"""Calcule la taille totale d'un répertoire en MB"""
total = 0
for dirpath, dirnames, filenames in os.walk(path):
for filename in filenames:
filepath = os.path.join(dirpath, filename)
if os.path.exists(filepath):
total += os.path.getsize(filepath)
return total / (1024 * 1024) # Convert to MB
output_paths = {
"CSV (uncompressed)": f"{OUTPUT_DIR}/fact_events_csv",
"CSV (Snappy)": f"{OUTPUT_DIR}/fact_events_csv_snappy",
"Parquet": f"{OUTPUT_DIR}/fact_events_parquet",
}
sizes = {}
for name, path in output_paths.items():
try:
size_mb = get_dir_size(path)
sizes[name] = size_mb
print(f"\n{name}:")
print(f" Size: {size_mb:.4f} MB")
except Exception as e:
print(f"\n{name}: Error - {e}")
# Calcule les ratios
if "Parquet" in sizes and sizes["Parquet"] > 0:
csv_ratio = sizes.get("CSV (uncompressed)", 0) / sizes["Parquet"]
snappy_ratio = sizes.get("CSV (Snappy)", 0) / sizes["Parquet"]
print("\n" + "=" * 70)
print("📈 COMPRESSION RATIOS (vs Parquet)")
print("=" * 70)
print(f"CSV vs Parquet: {csv_ratio:.1f}x larger")
print(f"CSV Snappy vs Parquet: {snappy_ratio:.1f}x larger")
print("=" * 70)
print(f"\n✅ Total data output: {sum(sizes.values()):.4f} MB")
====================================================================== 📊 FILE SIZE COMPARISON ====================================================================== CSV (uncompressed): Size: 0.0014 MB CSV (Snappy): Size: 0.0005 MB Parquet: Size: 0.0035 MB ====================================================================== 📈 COMPRESSION RATIOS (vs Parquet) ====================================================================== CSV vs Parquet: 0.4x larger CSV Snappy vs Parquet: 0.2x larger ====================================================================== ✅ Total data output: 0.0054 MB
In [19]:
# Cell 18: Spark Execution Plans
print("\n" + "=" * 70)
print("📋 SPARK EXECUTION PLANS")
print("=" * 70)
print("\n✅ Transform Plan (events_clean):")
print("-" * 70)
events_clean.explain(mode="formatted")
print("\n\n✅ Join & Aggregate Plan (fact_events):")
print("-" * 70)
fact_events.explain(mode="formatted")
======================================================================
📋 SPARK EXECUTION PLANS
======================================================================
✅ Transform Plan (events_clean):
----------------------------------------------------------------------
== Physical Plan ==
* Project (3)
+- * Filter (2)
+- Scan csv (1)
(1) Scan csv
Output [5]: [event_time#97, event_type#98, session_id#99, product_id#100, price#101]
Batched: false
Location: InMemoryFileIndex [file:/home/bibawandaogo/data engineering 1/lab2_data/events.csv]
PushedFilters: [IsNotNull(event_time), IsNotNull(session_id), IsNotNull(product_id), Or(IsNull(price),GreaterThanOrEqual(price,0.0)), In(event_type, [cart,purchase,remove,view]), LessThan(event_time,2025-12-09 00:00:00.0), Or(IsNull(price),LessThanOrEqual(price,29024.44999999999))]
ReadSchema: struct<event_time:timestamp,event_type:string,session_id:string,product_id:string,price:double>
(2) Filter [codegen id : 1]
Input [5]: [event_time#97, event_type#98, session_id#99, product_id#100, price#101]
Condition : ((((((isnotnull(event_time#97) AND isnotnull(session_id#99)) AND isnotnull(product_id#100)) AND (isnull(price#101) OR (price#101 >= 0.0))) AND event_type#98 IN (view,cart,purchase,remove)) AND (event_time#97 < 2025-12-09 00:00:00)) AND (isnull(price#101) OR (price#101 <= 29024.44999999999)))
(3) Project [codegen id : 1]
Output [6]: [event_time#97, event_type#98, session_id#99, product_id#100, price#101, cast(event_time#97 as date) AS event_date#567]
Input [5]: [event_time#97, event_type#98, session_id#99, product_id#100, price#101]
✅ Join & Aggregate Plan (fact_events):
----------------------------------------------------------------------
== Physical Plan ==
AdaptiveSparkPlan (75)
+- Project (74)
+- BroadcastNestedLoopJoin LeftOuter BuildRight (73)
:- Project (65)
: +- BroadcastHashJoin LeftOuter BuildRight (64)
: :- Project (57)
: : +- BroadcastHashJoin LeftOuter BuildRight (56)
: : :- Project (49)
: : : +- BroadcastHashJoin LeftOuter BuildRight (48)
: : : :- Project (43)
: : : : +- SortMergeJoin LeftOuter (42)
: : : : :- Sort (10)
: : : : : +- Exchange (9)
: : : : : +- Project (8)
: : : : : +- BroadcastHashJoin LeftOuter BuildRight (7)
: : : : : :- Project (3)
: : : : : : +- Filter (2)
: : : : : : +- Scan csv (1)
: : : : : +- BroadcastExchange (6)
: : : : : +- Filter (5)
: : : : : +- Scan csv (4)
: : : : +- Sort (41)
: : : : +- Exchange (40)
: : : : +- Project (39)
: : : : +- Filter (38)
: : : : +- Window (37)
: : : : +- Sort (36)
: : : : +- Exchange (35)
: : : : +- Project (34)
: : : : +- BroadcastHashJoin LeftOuter BuildRight (33)
: : : : :- Project (25)
: : : : : +- BroadcastHashJoin LeftOuter BuildRight (24)
: : : : : :- Project (16)
: : : : : : +- BroadcastHashJoin LeftOuter BuildRight (15)
: : : : : : :- Scan csv (11)
: : : : : : +- BroadcastExchange (14)
: : : : : : +- Filter (13)
: : : : : : +- Scan csv (12)
: : : : : +- BroadcastExchange (23)
: : : : : +- Project (22)
: : : : : +- Filter (21)
: : : : : +- Window (20)
: : : : : +- Sort (19)
: : : : : +- Exchange (18)
: : : : : +- Scan csv (17)
: : : : +- BroadcastExchange (32)
: : : : +- Project (31)
: : : : +- Filter (30)
: : : : +- Window (29)
: : : : +- Sort (28)
: : : : +- Exchange (27)
: : : : +- Scan csv (26)
: : : +- BroadcastExchange (47)
: : : +- Project (46)
: : : +- Generate (45)
: : : +- Scan OneRowRelation (44)
: : +- BroadcastExchange (55)
: : +- Filter (54)
: : +- Window (53)
: : +- Sort (52)
: : +- Exchange (51)
: : +- Scan csv (50)
: +- BroadcastExchange (63)
: +- Project (62)
: +- Window (61)
: +- Sort (60)
: +- Exchange (59)
: +- Scan csv (58)
+- BroadcastExchange (72)
+- Project (71)
+- Filter (70)
+- Window (69)
+- Sort (68)
+- Exchange (67)
+- Scan ExistingRDD (66)
(1) Scan csv
Output [5]: [event_time#97, event_type#98, session_id#99, product_id#100, price#101]
Batched: false
Location: InMemoryFileIndex [file:/home/bibawandaogo/data engineering 1/lab2_data/events.csv]
PushedFilters: [IsNotNull(event_time), IsNotNull(session_id), IsNotNull(product_id), Or(IsNull(price),GreaterThanOrEqual(price,0.0)), In(event_type, [cart,purchase,remove,view]), LessThan(event_time,2025-12-09 00:00:00.0), Or(IsNull(price),LessThanOrEqual(price,29024.44999999999))]
ReadSchema: struct<event_time:timestamp,event_type:string,session_id:string,product_id:string,price:double>
(2) Filter
Input [5]: [event_time#97, event_type#98, session_id#99, product_id#100, price#101]
Condition : ((((((isnotnull(event_time#97) AND isnotnull(session_id#99)) AND isnotnull(product_id#100)) AND (isnull(price#101) OR (price#101 >= 0.0))) AND event_type#98 IN (view,cart,purchase,remove)) AND (event_time#97 < 2025-12-09 00:00:00)) AND (isnull(price#101) OR (price#101 <= 29024.44999999999)))
(3) Project
Output [6]: [event_time#97, event_type#98, session_id#99, product_id#100, price#101, cast(event_time#97 as date) AS event_date#567]
Input [5]: [event_time#97, event_type#98, session_id#99, product_id#100, price#101]
(4) Scan csv
Output [2]: [session_id#37, user_id#38]
Batched: false
Location: InMemoryFileIndex [file:/home/bibawandaogo/data engineering 1/lab2_data/session.csv]
PushedFilters: [IsNotNull(session_id)]
ReadSchema: struct<session_id:string,user_id:string>
(5) Filter
Input [2]: [session_id#37, user_id#38]
Condition : isnotnull(session_id#37)
(6) BroadcastExchange
Input [2]: [session_id#37, user_id#38]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false]),false), [plan_id=29472]
(7) BroadcastHashJoin
Left keys [1]: [session_id#99]
Right keys [1]: [session_id#37]
Join type: LeftOuter
Join condition: None
(8) Project
Output [7]: [session_id#99, event_time#97, event_type#98, product_id#100, price#101, event_date#567, user_id#38]
Input [8]: [event_time#97, event_type#98, session_id#99, product_id#100, price#101, event_date#567, session_id#37, user_id#38]
(9) Exchange
Input [7]: [session_id#99, event_time#97, event_type#98, product_id#100, price#101, event_date#567, user_id#38]
Arguments: hashpartitioning(product_id#100, 200), ENSURE_REQUIREMENTS, [plan_id=29504]
(10) Sort
Input [7]: [session_id#99, event_time#97, event_type#98, product_id#100, price#101, event_date#567, user_id#38]
Arguments: [product_id#100 ASC NULLS FIRST], false, 0
(11) Scan csv
Output [4]: [product_id#56, brand#57, category#58, product_name#59]
Batched: false
Location: InMemoryFileIndex [file:/home/bibawandaogo/data engineering 1/lab2_data/product.csv]
ReadSchema: struct<product_id:string,brand:string,category:string,product_name:string>
(12) Scan csv
Output [2]: [category#77, product_name#78]
Batched: false
Location: InMemoryFileIndex [file:/home/bibawandaogo/data engineering 1/lab2_data/product_name.csv]
PushedFilters: [IsNotNull(category), IsNotNull(product_name)]
ReadSchema: struct<category:string,product_name:string>
(13) Filter
Input [2]: [category#77, product_name#78]
Condition : (isnotnull(category#77) AND isnotnull(product_name#78))
(14) BroadcastExchange
Input [2]: [category#77, product_name#78]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false], input[1, string, false]),false), [plan_id=29475]
(15) BroadcastHashJoin
Left keys [2]: [category#58, product_name#59]
Right keys [2]: [category#77, product_name#78]
Join type: LeftOuter
Join condition: None
(16) Project
Output [3]: [product_id#56, brand#57, category#58]
Input [6]: [product_id#56, brand#57, category#58, product_name#59, category#77, product_name#78]
(17) Scan csv
Output [1]: [category#119]
Batched: false
Location: InMemoryFileIndex [file:/home/bibawandaogo/data engineering 1/lab2_data/category.csv]
ReadSchema: struct<category:string>
(18) Exchange
Input [1]: [category#119]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=29478]
(19) Sort
Input [1]: [category#119]
Arguments: [category#119 ASC NULLS FIRST], false, 0
(20) Window
Input [1]: [category#119]
Arguments: [dense_rank(category#119) windowspecdefinition(category#119 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS category_key#295], [category#119 ASC NULLS FIRST]
(21) Filter
Input [2]: [category#119, category_key#295]
Condition : isnotnull(category#119)
(22) Project
Output [2]: [category_key#295, category#119 AS category_code#304]
Input [2]: [category#119, category_key#295]
(23) BroadcastExchange
Input [2]: [category_key#295, category_code#304]
Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=29484]
(24) BroadcastHashJoin
Left keys [1]: [category#58]
Right keys [1]: [category_code#304]
Join type: LeftOuter
Join condition: None
(25) Project
Output [3]: [product_id#56, brand#57, category_key#295]
Input [5]: [product_id#56, brand#57, category#58, category_key#295, category_code#304]
(26) Scan csv
Output [1]: [brand#138]
Batched: false
Location: InMemoryFileIndex [file:/home/bibawandaogo/data engineering 1/lab2_data/brand.csv]
ReadSchema: struct<brand:string>
(27) Exchange
Input [1]: [brand#138]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=29487]
(28) Sort
Input [1]: [brand#138]
Arguments: [brand#138 ASC NULLS FIRST], false, 0
(29) Window
Input [1]: [brand#138]
Arguments: [dense_rank(brand#138) windowspecdefinition(brand#138 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS brand_key#264], [brand#138 ASC NULLS FIRST]
(30) Filter
Input [2]: [brand#138, brand_key#264]
Condition : isnotnull(brand#138)
(31) Project
Output [2]: [brand_key#264, brand#138 AS brand_code#273]
Input [2]: [brand#138, brand_key#264]
(32) BroadcastExchange
Input [2]: [brand_key#264, brand_code#273]
Arguments: HashedRelationBroadcastMode(List(input[1, string, true]),false), [plan_id=29493]
(33) BroadcastHashJoin
Left keys [1]: [brand#57]
Right keys [1]: [brand_code#273]
Join type: LeftOuter
Join condition: None
(34) Project
Output [3]: [product_id#56, category_key#295, brand_key#264]
Input [5]: [product_id#56, brand#57, category_key#295, brand_key#264, brand_code#273]
(35) Exchange
Input [3]: [product_id#56, category_key#295, brand_key#264]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=29497]
(36) Sort
Input [3]: [product_id#56, category_key#295, brand_key#264]
Arguments: [product_id#56 ASC NULLS FIRST], false, 0
(37) Window
Input [3]: [product_id#56, category_key#295, brand_key#264]
Arguments: [dense_rank(product_id#56) windowspecdefinition(product_id#56 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS product_key#328], [product_id#56 ASC NULLS FIRST]
(38) Filter
Input [4]: [product_id#56, category_key#295, brand_key#264, product_key#328]
Condition : isnotnull(product_id#56)
(39) Project
Output [4]: [product_id#56, product_key#328, brand_key#264, category_key#295]
Input [4]: [product_id#56, category_key#295, brand_key#264, product_key#328]
(40) Exchange
Input [4]: [product_id#56, product_key#328, brand_key#264, category_key#295]
Arguments: hashpartitioning(product_id#56, 200), ENSURE_REQUIREMENTS, [plan_id=29505]
(41) Sort
Input [4]: [product_id#56, product_key#328, brand_key#264, category_key#295]
Arguments: [product_id#56 ASC NULLS FIRST], false, 0
(42) SortMergeJoin
Left keys [1]: [product_id#100]
Right keys [1]: [product_id#56]
Join type: LeftOuter
Join condition: None
(43) Project
Output [9]: [session_id#99, event_time#97, event_type#98, price#101, event_date#567, user_id#38, product_key#328, brand_key#264, category_key#295]
Input [11]: [session_id#99, event_time#97, event_type#98, product_id#100, price#101, event_date#567, user_id#38, product_id#56, product_key#328, brand_key#264, category_key#295]
(44) Scan OneRowRelation
Output: []
Arguments: ParallelCollectionRDD[237] at count at NativeMethodAccessorImpl.java:0, OneRowRelation, UnknownPartitioning(0)
(45) Generate
Input: []
Arguments: explode(org.apache.spark.sql.catalyst.expressions.UnsafeArrayData@3966a5e), false, [date#444]
(46) Project
Output [2]: [date#444, (((year(date#444) * 10000) + (month(date#444) * 100)) + dayofmonth(date#444)) AS date_key#454]
Input [1]: [date#444]
(47) BroadcastExchange
Input [2]: [date#444, date_key#454]
Arguments: HashedRelationBroadcastMode(List(input[0, date, false]),false), [plan_id=29511]
(48) BroadcastHashJoin
Left keys [1]: [event_date#567]
Right keys [1]: [date#444]
Join type: LeftOuter
Join condition: None
(49) Project
Output [10]: [session_id#99, event_time#97, event_type#98, price#101, event_date#567, user_id#38, product_key#328, brand_key#264, category_key#295, date_key#454]
Input [11]: [session_id#99, event_time#97, event_type#98, price#101, event_date#567, user_id#38, product_key#328, brand_key#264, category_key#295, date#444, date_key#454]
(50) Scan csv
Output [1]: [user_id#17]
Batched: false
Location: InMemoryFileIndex [file:/home/bibawandaogo/data engineering 1/lab2_data/user.csv]
ReadSchema: struct<user_id:string>
(51) Exchange
Input [1]: [user_id#17]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=29514]
(52) Sort
Input [1]: [user_id#17]
Arguments: [user_id#17 ASC NULLS FIRST], false, 0
(53) Window
Input [1]: [user_id#17]
Arguments: [dense_rank(user_id#17) windowspecdefinition(user_id#17 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS user_key#193], [user_id#17 ASC NULLS FIRST]
(54) Filter
Input [2]: [user_id#17, user_key#193]
Condition : isnotnull(user_id#17)
(55) BroadcastExchange
Input [2]: [user_id#17, user_key#193]
Arguments: HashedRelationBroadcastMode(List(input[0, string, false]),false), [plan_id=29519]
(56) BroadcastHashJoin
Left keys [1]: [user_id#38]
Right keys [1]: [user_id#17]
Join type: LeftOuter
Join condition: None
(57) Project
Output [10]: [session_id#99, event_time#97, event_type#98, price#101, event_date#567, product_key#328, brand_key#264, category_key#295, date_key#454, user_key#193]
Input [12]: [session_id#99, event_time#97, event_type#98, price#101, event_date#567, user_id#38, product_key#328, brand_key#264, category_key#295, date_key#454, user_id#17, user_key#193]
(58) Scan csv
Output [2]: [user_id#696, birthdate#698]
Batched: false
Location: InMemoryFileIndex [file:/home/bibawandaogo/data engineering 1/lab2_data/user.csv]
ReadSchema: struct<user_id:string,birthdate:date>
(59) Exchange
Input [2]: [user_id#696, birthdate#698]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=29522]
(60) Sort
Input [2]: [user_id#696, birthdate#698]
Arguments: [user_id#696 ASC NULLS FIRST], false, 0
(61) Window
Input [2]: [user_id#696, birthdate#698]
Arguments: [dense_rank(user_id#696) windowspecdefinition(user_id#696 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS user_key#705], [user_id#696 ASC NULLS FIRST]
(62) Project
Output [2]: [user_key#705, birthdate#698]
Input [3]: [user_id#696, birthdate#698, user_key#705]
(63) BroadcastExchange
Input [2]: [user_key#705, birthdate#698]
Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=29527]
(64) BroadcastHashJoin
Left keys [1]: [user_key#193]
Right keys [1]: [user_key#705]
Join type: LeftOuter
Join condition: None
(65) Project
Output [10]: [user_key#193, session_id#99, event_time#97, event_type#98, price#101, product_key#328, brand_key#264, category_key#295, date_key#454, FLOOR((months_between(cast(event_date#567 as timestamp), cast(birthdate#698 as timestamp), true, Some(Europe/Paris)) / 12.0)) AS age_on_event#710L]
Input [12]: [session_id#99, event_time#97, event_type#98, price#101, event_date#567, product_key#328, brand_key#264, category_key#295, date_key#454, user_key#193, user_key#705, birthdate#698]
(66) Scan ExistingRDD
Output [3]: [age_band#230, min_age#231L, max_age#232L]
Arguments: [age_band#230, min_age#231L, max_age#232L], MapPartitionsRDD[137] at applySchemaToPythonRDD at NativeMethodAccessorImpl.java:0, ExistingRDD, UnknownPartitioning(0)
(67) Exchange
Input [3]: [age_band#230, min_age#231L, max_age#232L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=29530]
(68) Sort
Input [3]: [age_band#230, min_age#231L, max_age#232L]
Arguments: [age_band#230 ASC NULLS FIRST], false, 0
(69) Window
Input [3]: [age_band#230, min_age#231L, max_age#232L]
Arguments: [dense_rank(age_band#230) windowspecdefinition(age_band#230 ASC NULLS FIRST, specifiedwindowframe(RowFrame, unboundedpreceding$(), currentrow$())) AS age_key#233], [age_band#230 ASC NULLS FIRST]
(70) Filter
Input [4]: [age_band#230, min_age#231L, max_age#232L, age_key#233]
Condition : (isnotnull(min_age#231L) AND isnotnull(max_age#232L))
(71) Project
Output [3]: [age_key#233, min_age#231L, max_age#232L]
Input [4]: [age_band#230, min_age#231L, max_age#232L, age_key#233]
(72) BroadcastExchange
Input [3]: [age_key#233, min_age#231L, max_age#232L]
Arguments: IdentityBroadcastMode, [plan_id=29536]
(73) BroadcastNestedLoopJoin
Join type: LeftOuter
Join condition: ((age_on_event#710L > min_age#231L) AND (age_on_event#710L <= max_age#232L))
(74) Project
Output [10]: [date_key#454, user_key#193, age_key#233, product_key#328, brand_key#264, category_key#295, session_id#99, event_time#97, event_type#98, price#101]
Input [13]: [user_key#193, session_id#99, event_time#97, event_type#98, price#101, product_key#328, brand_key#264, category_key#295, date_key#454, age_on_event#710L, age_key#233, min_age#231L, max_age#232L]
(75) AdaptiveSparkPlan
Output [10]: [date_key#454, user_key#193, age_key#233, product_key#328, brand_key#264, category_key#295, session_id#99, event_time#97, event_type#98, price#101]
Arguments: isFinalPlan=false
25/12/08 23:25:56 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:25:56 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:25:56 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:25:56 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:25:56 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:25:56 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:25:56 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:25:56 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:25:56 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:25:56 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:25:56 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:25:56 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
In [20]:
# Cell 19: Final Summary
print("\n\n" + "=" * 70)
print("🎉 LAB 2 ASSIGNMENT - COMPLETE SUMMARY")
print("=" * 70)
print("\n📊 DATA WAREHOUSE STAR SCHEMA:")
print("-" * 70)
print(f"dim_user: {dim_user.count():>6} rows | FK in fact_events")
print(f"dim_age: {dim_age.count():>6} rows | FK in fact_events")
print(f"dim_brand: {dim_brand.count():>6} rows | FK in fact_events")
print(f"dim_category: {dim_category.count():>6} rows | FK in fact_events")
print(f"dim_product: {dim_product.count():>6} rows | FK in fact_events")
print(f"dim_date: {dim_date.count():>6} rows | FK in fact_events")
print(f"{'─' * 70}")
print(f"fact_events: {fact_events.count():>6} rows | Main fact table")
print("\n🔍 QUALITY GATES:")
print("-" * 70)
print("✅ Gate 1: Row count non-zero")
print("✅ Gate 2: Null rate thresholds")
print("✅ Gate 3: Referential integrity")
print("\n💾 OUTPUTS:")
print("-" * 70)
print(f"CSV uncompressed: {sizes.get('CSV (uncompressed)', 0):.4f} MB")
print(f"CSV Snappy: {sizes.get('CSV (Snappy)', 0):.4f} MB")
print(f"Parquet: {sizes.get('Parquet', 0):.4f} MB")
print("\n⚙️ SPARK CONFIG:")
print("-" * 70)
print(f"Version: {spark.version}")
print(f"Master: {spark.sparkContext.master}")
print(f"Driver Memory: 8g")
print(f"Shuffle Partitions: 200")
print(f"Adaptive Execution: Enabled")
print("\n📝 KEY INSIGHTS:")
print("-" * 70)
print("1. Parquet is much smaller than CSV formats")
print(" → Columnar storage compresses better")
print(" → Better for analytical queries")
print("\n2. Quality gates ensure data integrity")
print(" → All foreign keys validated")
print(" → Null rates within thresholds")
print("\n3. Built-in functions used (no UDFs)")
print(" → F.months_between for age calculation")
print(" → F.dense_rank for surrogate keys")
print(" → Better performance than custom code")
print("\n" + "=" * 70)
print("✅ ALL TASKS COMPLETED SUCCESSFULLY!")
print("=" * 70)
====================================================================== 🎉 LAB 2 ASSIGNMENT - COMPLETE SUMMARY ====================================================================== 📊 DATA WAREHOUSE STAR SCHEMA: ---------------------------------------------------------------------- dim_user: 10 rows | FK in fact_events dim_age: 10 rows | FK in fact_events dim_brand: 5 rows | FK in fact_events dim_category: 5 rows | FK in fact_events dim_product: 10 rows | FK in fact_events dim_date: 4 rows | FK in fact_events ──────────────────────────────────────────────────────────────────────
25/12/08 23:26:20 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:26:20 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:26:20 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:26:20 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:26:20 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:26:20 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:26:20 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation. 25/12/08 23:26:20 WARN WindowExec: No Partition Defined for Window operation! Moving all data to a single partition, this can cause serious performance degradation.
fact_events: 20 rows | Main fact table 🔍 QUALITY GATES: ---------------------------------------------------------------------- ✅ Gate 1: Row count non-zero ✅ Gate 2: Null rate thresholds ✅ Gate 3: Referential integrity 💾 OUTPUTS: ---------------------------------------------------------------------- CSV uncompressed: 0.0014 MB CSV Snappy: 0.0005 MB Parquet: 0.0035 MB ⚙️ SPARK CONFIG: ---------------------------------------------------------------------- Version: 4.0.1 Master: local[*] Driver Memory: 8g Shuffle Partitions: 200 Adaptive Execution: Enabled 📝 KEY INSIGHTS: ---------------------------------------------------------------------- 1. Parquet is much smaller than CSV formats → Columnar storage compresses better → Better for analytical queries 2. Quality gates ensure data integrity → All foreign keys validated → Null rates within thresholds 3. Built-in functions used (no UDFs) → F.months_between for age calculation → F.dense_rank for surrogate keys → Better performance than custom code ====================================================================== ✅ ALL TASKS COMPLETED SUCCESSFULLY! ======================================================================