我们讲到了如何启动Master和Worker，还讲到了如何回收资源。但是，我们没有将AppClient是如何启动的，其实它们的启动也涉及到了资源是如何调度的。这篇博文，我们就来讲一下AppClient的启动和逻辑与物理上的资源调度。
启动AppClient
调用栈如下：
StandaloneSchedulerBackend.start
StandaloneAppClient.start
StandaloneAppClient.registerWithMaster
StandaloneAppClient.tryRegisterAllMasters
StandaloneAppClient.ClientEndpoint.onStart
Master.receive
Master.createApplication
Master.registerApplication
StandaloneAppClient.ClientEndpoint.receive
StandaloneSchedulerBackend.start
在Standalone模式下，SparkContext中的backend是StandaloneSchedulerBackend。在StandaloneSchedulerBackend.start中可以看到：
*** val appUIAddress = sc.ui.map(_.appUIAddress).getOrElse("") val coresPerExecutor = conf.getOption("spark.executor.cores").map(_.toInt)
val initialExecutorLimit = if (Utils.isDynamicAllocationEnabled(conf)) { Some(0)
} else { None
} val appDesc = new ApplicationDescription(sc.appName, maxCores, sc.executorMemory, command,
appUIAddress, sc.eventLogDir, sc.eventLogCodec, coresPerExecutor, initialExecutorLimit) //创建AppClient
client = new StandaloneAppClient(sc.env.rpcEnv, masters, appDesc, this, conf) //启动AppClient
client.start()
***StandaloneAppClient.start
def start() { //生成了ClientEndpoint，于是调用其onStart
endpoint.set(rpcEnv.setupEndpoint("AppClient", new ClientEndpoint(rpcEnv)))
}StandaloneAppClient.ClientEndpoint.onStart
调用registerWithMaster
override def onStart(): Unit = { try {
registerWithMaster(1)
} catch { case e: Exception =>
logWarning("Failed to connect to master", e)
markDisconnected()
stop()
}
}StandaloneAppClient.registerWithMaster
private def registerWithMaster(nthRetry: Int) { //向所有的Master注册当前App
//一旦成功连接的一个master，其他将被取消
registerMasterFutures.set(tryRegisterAllMasters())
registrationRetryTimer.set(registrationRetryThread.schedule(new Runnable { override def run(): Unit = { if (registered.get) {
registerMasterFutures.get.foreach(_.cancel(true))
registerMasterThreadPool.shutdownNow()
}
//若达到最大尝试次数，则标志死亡，默认为3
else if (nthRetry >= REGISTRATION_RETRIES) {
markDead("All masters are unresponsive! Giving up.")
} else {
registerMasterFutures.get.foreach(_.cancel(true))
registerWithMaster(nthRetry + 1)
}
}
}, REGISTRATION_TIMEOUT_SECONDS, TimeUnit.SECONDS))
}StandaloneAppClient.tryRegisterAllMasters
给Master发送RegisterApplication信号：
private def tryRegisterAllMasters(): Array[JFuture[_]] = { for (masterAddress <- masterRpcAddresses) yield {
registerMasterThreadPool.submit(new Runnable { override def run(): Unit = try { if (registered.get) { return
}
logInfo("Connecting to master " + masterAddress.toSparkURL + "...") val masterRef = rpcEnv.setupEndpointRef(masterAddress, Master.ENDPOINT_NAME)
masterRef.send(RegisterApplication(appDescription, self))
} catch { case ie: InterruptedException => // Cancelled
case NonFatal(e) => logWarning(s"Failed to connect to master $masterAddress", e)
}
})
}
}Master.receive
Master.receive接收并处理RegisterApplication信号
case RegisterApplication(description, driver) => // 若之前注册过
if (state == RecoveryState.STANDBY) { // 忽略
} else {
logInfo("Registering app " + description.name) //创建app
val app = createApplication(description, driver) //注册app
registerApplication(app)
logInfo("Registered app " + description.name + " with ID " + app.id) //持久化
persistenceEngine.addApplication(app) //回复RegisteredApplication信号
driver.send(RegisteredApplication(app.id, self)) //资源调度
schedule()
}让我们深入来看下Master是如何注册app的。
Master.createApplication
先创建app：
private def createApplication(desc: ApplicationDescription, driver: RpcEndpointRef): ApplicationInfo = { val now = System.currentTimeMillis() val date = new Date(now) //根据日期生成appId
val appId = newApplicationId(date) //传入 时间，appId, 描述信息, 日期, driver, 默认核数，
//生成app信息
new ApplicationInfo(now, appId, desc, date, driver, defaultCores)
}Master.registerApplication
再注册app：
private def registerApplication(app: ApplicationInfo): Unit = { //若已有这个app地址，
//则返回
val appAddress = app.driver.address if (addressToApp.contains(appAddress)) {
logInfo("Attempted to re-register application at same address: " + appAddress) return
} //向 applicationMetricsSystem 注册appSource
applicationMetricsSystem.registerSource(app.appSource) //将app加入到 集合
//HashSet[ApplicationInfo]
apps += app //更新 id到App
//HashMap[String, ApplicationInfo]
idToApp(app.id) = app //更新 endpoint到App
// HashMap[RpcEndpointRef, ApplicationInfo]
endpointToApp(app.driver) = app //更新 address到App
// HashMap[RpcAddress, ApplicationInfo]
addressToApp(appAddress) = app // 加入到等待数组中
//ArrayBuffer[ApplicationInfo]
waitingApps += app if (reverseProxy) {
webUi.addProxyTargets(app.id, app.desc.appUiUrl)
}
}StandaloneAppClient.ClientEndpoint.receive
case RegisteredApplication(appId_, masterRef) => //这里的代码有两个缺陷： //1. 一个Master可能接收到多个注册请求， // 并且回复多个RegisteredApplication信号， //这会导致网络不稳定。 //2.若master正在变化， //则会接收到多个RegisteredApplication信号 //设置appId appId.set(appId_) //编辑已经注册 registered.set(true) //创建master信息 master = Some(masterRef) //绑定监听 listener.connected(appId.get)
逻辑资源调度
我们可以看到在上一章，Master.receive接收并处理RegisterApplication信号时的最后一行代码：
//资源调度 schedule()
下面，我们就来讲讲资源调度。
调用栈如下：
Master.schedule
Master.startExecutorsOnWorkers
Master.scheduleExecutorsOnWorkers
Master.allocateWorkerResourceToExecutors
Master.schedule
该方法主要来在等待的app之间调度资源。每次有新的app加入或者可用资源改变的时候，这个方法都会被调用：
private def schedule(): Unit = { if (state != RecoveryState.ALIVE) { return
} // 得到活的Worker，
// 并打乱它们
val shuffledAliveWorkers = Random.shuffle(workers.toSeq.filter(_.state == WorkerState.ALIVE)) // worker数量
val numWorkersAlive = shuffledAliveWorkers.size var curPos = 0
//为driver分配资源，
//该调度策略为FIFO的策略，
//先来的driver会先满足其资源所需的条件
for (driver <- waitingDrivers.toList) {
var launched = false
var numWorkersVisited = 0
while (numWorkersVisited < numWorkersAlive && !launched) { val worker = shuffledAliveWorkers(curPos)
numWorkersVisited += 1
if (worker.memoryFree >= driver.desc.mem && worker.coresFree >= driver.desc.cores) {
launchDriver(worker, driver)
waitingDrivers -= driver
launched = true
}
curPos = (curPos + 1) % numWorkersAlive
}
} //启动worker上的executor
startExecutorsOnWorkers()
}Master.startExecutorsOnWorkers
接下来我们来看下executor的启动：
private def startExecutorsOnWorkers(): Unit = { // 这里还是使用的FIFO的调度方式
for (app <- waitingApps if app.coresLeft > 0) { val coresPerExecutor: Option[Int] = app.desc.coresPerExecutor // 过滤掉资源不够启动executor的worker
// 并按资源逆序排序
val usableWorkers = workers.toArray.filter(_.state == WorkerState.ALIVE)
.filter(worker => worker.memoryFree >= app.desc.memoryPerExecutorMB &&
worker.coresFree >= coresPerExecutor.getOrElse(1))
.sortBy(_.coresFree).reverse //调度worker上的executor，
//确定在每个worker上给这个app分配多少核
val assignedCores = scheduleExecutorsOnWorkers(app, usableWorkers, spreadOutApps) //分配
for (pos <- 0 until usableWorkers.length if assignedCores(pos) > 0) {
allocateWorkerResourceToExecutors(
app, assignedCores(pos), coresPerExecutor, usableWorkers(pos))
}
}
}Master.scheduleExecutorsOnWorkers
接下来我们就来讲讲核心的worker上的executor资源调度。在将现在的Spark代码之前，我们看看在Spark1.4之前，这部分逻辑是如何实现的：
*** val numUsable = usableWorkers.length // 用来记录每个worker已经分配的核数
val assigned = new Array[Int](numUsable)
var toAssign = math.min(app.coresLeft, usableWorkers.map(_.coresFree).sum) var pos = 0
while (toAssign > 0) { //遍历worker，
//若当前worker还存在资源，
//则分配掉1个核。
//直到workers的资源全都被分配掉，
//或者是app所需要的资源被满足。
if (usableWorkers(pos).coresFree - assigned(pos) > 0) {
toAssign -= 1
assigned(pos) += 1
}
pos = (pos + 1) % numUsable
}
***在Spark1.4的时候，这段代码被修改了。我们来想一下，以上代码有什么问题？
问题就在于，core是一个一个的被分配的。设想，一个集群中有4 worker，每个worker有16个core。用户想启动3个executor，且每个executor拥有16个core。于是，他会这样配置参数：
spark.cores.max = 48 spark.executor.cores = 16
显然，我们集群的资源是能满足用户的需求的。但如果一次只能分配一个core，那最终的结果是每个worker上都分配了12个core。由于12 < 16, 所有没有一个executor能够启动。
下面，我们回过头来看现在的源码中是如何实现这部分逻辑的：
private def scheduleExecutorsOnWorkers(
app: ApplicationInfo,
usableWorkers: Array[WorkerInfo],
spreadOutApps: Boolean): Array[Int] = { val coresPerExecutor = app.desc.coresPerExecutor val minCoresPerExecutor = coresPerExecutor.getOrElse(1) val oneExecutorPerWorker = coresPerExecutor.isEmpty val memoryPerExecutor = app.desc.memoryPerExecutorMB val numUsable = usableWorkers.length // 用来记录每个worker已经分配的核数
val assignedCores = new Array[Int](numUsable)
// 用来记录每个worker已经分配的executor数
val assignedExecutors = new Array[Int](numUsable) // 剩余总共资源
var coresToAssign = math.min(app.coresLeft, usableWorkers.map(_.coresFree).sum) //判断是否能启动Executor
def canLaunchExecutor(pos: Int): Boolean = { //先省略
} //过滤去能启动executor的Worker
var freeWorkers = (0 until numUsable).filter(canLaunchExecutor) //调度资源，
//直到worker上的executor被分配完
while (freeWorkers.nonEmpty) {
freeWorkers.foreach { pos => var keepScheduling = true
while (keepScheduling && canLaunchExecutor(pos)) { // minCoresPerExecutor 是用户设置的 spark.executor.cores
coresToAssign -= minCoresPerExecutor
assignedCores(pos) += minCoresPerExecutor // 若用户没有设置 spark.executor.cores
// 则oneExecutorPerWorker就为True
// 也就是说，assignedCores中的core都被一个executor使用
// 若用户设置了spark.executor.cores，
// 则该Worker的assignedExecutors会加1
if (oneExecutorPerWorker) {
assignedExecutors(pos) = 1
} else {
assignedExecutors(pos) += 1
}
//资源分配算法有两种：
// 1. 尽量打散，将一个app尽可能的分配到不同的节点上，
// 这有利于充分利用集群的资源，
// 在这种情况下，spreadOutApps设为True，
// 于是，该worker分配好了一个executor之后就退出循环
// 轮询到下一个worker
// 2. 尽量集中，将一个app尽可能的分配到同一个的节点上，
// 这适合cpu密集型而内存占用比较少的app
// 在这种情况下，spreadOutApps设为False，
// 于是，继续下一轮的循环
// 在该Worker上分配executor
if (spreadOutApps) {
keepScheduling = false
}
}
}
freeWorkers = freeWorkers.filter(canLaunchExecutor)
}
assignedCores
}接下来看下该函数的内部函数canLaunchExecutor：
def canLaunchExecutor(pos: Int): Boolean = { // 条件1 ：若集群剩余core >= spark.executor.cores
val keepScheduling = coresToAssign >= minCoresPerExecutor // 条件2： 若该Worker上的剩余core >= spark.executor.cores
val enoughCores = usableWorkers(pos).coresFree - assignedCores(pos) >= minCoresPerExecutor
// 条件3： 若设置了spark.executor.cores
// 或者 该Worker还未分配executor
val launchingNewExecutor = !oneExecutorPerWorker || assignedExecutors(pos) == 0
if (launchingNewExecutor) { val assignedMemory = assignedExecutors(pos) * memoryPerExecutor // 条件4：若该Worker上的剩余内存 >= spark.executor.memory
val enoughMemory = usableWorkers(pos).memoryFree - assignedMemory >= memoryPerExecutor // 条件5： 若分配了该executor后，
// 总共分配的core数量 <= spark.cores.max
val underLimit = assignedExecutors.sum + app.executors.size < app.executorLimit //若满足 条件3，
//且满足 条件1，条件2，条件4，条件5
//则返回True
keepScheduling && enoughCores && enoughMemory && underLimit
} else { //若不满足 条件3，
//即一个worker只有一个executor
//且满足 条件1，条件2
//也返回True。
// 返回后，不会增加 assignedExecutors
keepScheduling && enoughCores
}
}通过以上源码，我们可以清楚看到，Spark1.4以后新的逻辑实现其实就是将分配单位从原来的一个core，变为了一个executor（即spark.executor.cores）。而若一个worker上只有一个executor（即没有设置spark.executor.cores），那么就按照原来的逻辑实现。
值得我注意的是：
//直到worker上的executor被分配完 while (freeWorkers.nonEmpty)
一个app会尽可能的使用掉集群的所有资源，所以设置spark.cores.max参数是非常有必要的！
Master.allocateWorkerResourceToExecutors
现在我们回到上述提到的Master.startExecutorsOnWorkers中，深入allocateWorkerResourceToExecutors：
private def allocateWorkerResourceToExecutors(
app: ApplicationInfo,
assignedCores: Int,
coresPerExecutor: Option[Int],
worker: WorkerInfo): Unit = { // 该work上的executor数量
// 若没设置 spark.executor.cores
// 则为1
val numExecutors = coresPerExecutor.map { assignedCores / _ }.getOrElse(1) // 分配给一个executor的core数量
// 若没设置 spark.executor.cores
// 则为该worker上所分配的所有core是数量
val coresToAssign = coresPerExecutor.getOrElse(assignedCores) for (i <- 1 to numExecutors) { //创建该executor信息
//并把它加入到app信息中
//并返回executor信息
val exec = app.addExecutor(worker, coresToAssign) //启动
launchExecutor(worker, exec)
app.state = ApplicationState.RUNNING
}
}要注意的是
app.state = ApplicationState.RUNNING
这句代码并不是将该app从waitingApp队列中去除。若在该次资源调度中该app并没有启动足够的executor，等到集群资源变化时，会再次资源调度，在waitingApp中遍历到该app，其coresLeft > 0。
for (app <- waitingApps if app.coresLeft > 0)
我们这里做一个实验：
我们的实验集群是4*8核的集群：
这里写图片描述
第1个app，我们申请4个executor，该executor为4个core：
spark-shell --master spark://cdh03:7077 --total-executor-cores 4 --executor-cores 4
可以看到集群资源：
这里写图片描述
app1的executor：
这里写图片描述
第2个app，我们申请4个executor，该executor为6个core：
spark-shell --master spark://cdh03:7077 --total-executor-cores 24 --executor-cores 6
可以看到集群资源：
这里写图片描述
app2的executor：
这里写图片描述
我们可以看到，Spark只为app2分配了3个executor。
当我们把app1退出
会发现集群资源状态：
这里写图片描述
app2的executor：
这里写图片描述
会发现新增加了一个“ worker-20170102151129”的executor。
其实，只要集群中的app没结束，它们都会在waitingApps中，当该app结束时，才会将这个app从waitingApps中移除
def removeApplication(app: ApplicationInfo, state: ApplicationState.Value) {
***
waitingApps -= app
***
}物理资源调度与启动Executor
接下来，我们就来讲逻辑上资源调度完后，该如何物理上资源调度，即启动Executor。
这里写图片描述
调用栈如下：
Master.launchExecutor
Worker.receive
ExecutorRunner.start
ExecutorRunner.fetchAndRunExecutor
CoarseGrainedExecutorBackend.main
CoarseGrainedExecutorBackend.run
CoarseGrainedExecutorBackend.onStart
CoarseGrainedSchedulerBackend.DriverEndpoint.receiveAndReply
CoarseGrainedExecutorBackend.receive
Master.launchExecutor
private def launchExecutor(worker: WorkerInfo, exec: ExecutorDesc): Unit = {
logInfo("Launching executor " + exec.fullId + " on worker " + worker.id) //在worker信息中加入executor信息
worker.addExecutor(exec) //给worker发送LaunchExecutor信号
worker.endpoint.send(LaunchExecutor(masterUrl,
exec.application.id, exec.id, exec.application.desc, exec.cores, exec.memory)) //给driver发送ExecutorAdded信号
exec.application.driver.send( ExecutorAdded(exec.id, worker.id, worker.hostPort, exec.cores, exec.memory))
}Worker.receive
worker接收到LaunchExecutor信号后处理：
case LaunchExecutor(masterUrl, appId, execId, appDesc, cores_, memory_) => if (masterUrl != activeMasterUrl) {
logWarning("Invalid Master (" + masterUrl + ") attempted to launch executor.")
} else { try {
logInfo("Asked to launch executor %s/%d for %s".format(appId, execId, appDesc.name)) // 创建executor的工作目录
// shuffle持久化结果会存在这个目录下
// 节点应每块磁盘大小尽可能相同
// 并在配置中在每块磁盘上都设置SPARK_WORKER_DIR,
// 以增加IO性能
val executorDir = new File(workDir, appId + "/" + execId) if (!executorDir.mkdirs()) { throw new IOException("Failed to create directory " + executorDir)
} // 为app创建本地dir
// app完成后，此目录会被删除
val appLocalDirs = appDirectories.getOrElse(appId, Utils.getOrCreateLocalRootDirs(conf).map { dir => val appDir = Utils.createDirectory(dir, namePrefix = "executor") Utils.chmod700(appDir)
appDir.getAbsolutePath()
}.toSeq)
appDirectories(appId) = appLocalDirs //创建ExecutorRunner
val manager = new ExecutorRunner(
appId,
execId,
appDesc.copy(command = Worker.maybeUpdateSSLSettings(appDesc.command, conf)),
cores_,
memory_,
self,
workerId,
host,
webUi.boundPort,
publicAddress,
sparkHome,
executorDir,
workerUri,
conf,
appLocalDirs, ExecutorState.RUNNING)
executors(appId + "/" + execId) = manager //启动ExecutorRunner
manager.start()
coresUsed += cores_
memoryUsed += memory_ // 向Master发送ExecutorStateChanged信号
sendToMaster(ExecutorStateChanged(appId, execId, manager.state, None, None))
} catch { case e: Exception =>
logError(s"Failed to launch executor $appId/$execId for ${appDesc.name}.", e) if (executors.contains(appId + "/" + execId)) {
executors(appId + "/" + execId).kill()
executors -= appId + "/" + execId
}
sendToMaster(ExecutorStateChanged(appId, execId, ExecutorState.FAILED, Some(e.toString), None))
}
}ExecutorRunner.start
接下来我们深入看下ExecutorRunner
private[worker] def start() { //创建worker线程
workerThread = new Thread("ExecutorRunner for " + fullId) { override def run() { fetchAndRunExecutor() }
} //启动worker线程
workerThread.start() // 创建Shutdownhook线程
// 用于worker关闭时，杀掉executor
shutdownHook = ShutdownHookManager.addShutdownHook { () => if (state == ExecutorState.RUNNING) {
state = ExecutorState.FAILED
}
killProcess(Some("Worker shutting down")) }
}ExecutorRunner.fetchAndRunExecutor
workerThread执行主要是调用fetchAndRunExecutor，下面我们来看下这个方法：
private def fetchAndRunExecutor() { try { // 创建进程builder
val builder = CommandUtils.buildProcessBuilder(appDesc.command, new SecurityManager(conf),
memory, sparkHome.getAbsolutePath, substituteVariables) val command = builder.command() val formattedCommand = command.asScala.mkString("\"", "\" \"", "\"")
logInfo(s"Launch command: $formattedCommand") //创建进程builder执行目录
builder.directory(executorDir) //为进程builder设置环境变量
builder.environment.put("SPARK_EXECUTOR_DIRS", appLocalDirs.mkString(File.pathSeparator))
builder.environment.put("SPARK_LAUNCH_WITH_SCALA", "0") val baseUrl = if (conf.getBoolean("spark.ui.reverseProxy", false)) { s"/proxy/$workerId/logPage/?appId=$appId&executorId=$execId&logType="
} else { s"http://$publicAddress:$webUiPort/logPage/?appId=$appId&executorId=$execId&logType="
}
builder.environment.put("SPARK_LOG_URL_STDERR", s"${baseUrl}stderr")
builder.environment.put("SPARK_LOG_URL_STDOUT", s"${baseUrl}stdout")
//启动进程builder，创建进程
process = builder.start() val header = "Spark Executor Command: %s\n%s\n\n".format(
formattedCommand, "=" * 40) // 重定向它的stdout和stderr到文件中
val stdout = new File(executorDir, "stdout")
stdoutAppender = FileAppender(process.getInputStream, stdout, conf) val stderr = new File(executorDir, "stderr") Files.write(header, stderr, StandardCharsets.UTF_8)
stderrAppender = FileAppender(process.getErrorStream, stderr, conf) // 等待进程退出。
// 当driver通知该进程退出
// executor会退出并返回0或者非0的exitCode
val exitCode = process.waitFor()
state = ExecutorState.EXITED
val message = "Command exited with code " + exitCode // 给Worker发送ExecutorStateChanged信号
worker.send(ExecutorStateChanged(appId, execId, state, Some(message), Some(exitCode)))
} catch { case interrupted: InterruptedException =>
logInfo("Runner thread for executor " + fullId + " interrupted")
state = ExecutorState.KILLED
killProcess(None) case e: Exception =>
logError("Error running executor", e)
state = ExecutorState.FAILED
killProcess(Some(e.toString))
}
}
}CoarseGrainedExecutorBackend.main
builder start的是CoarseGrainedExecutorBackend实例进程，我们看下它的主函数：
def main(args: Array[String]) { var driverUrl: String = null
var executorId: String = null
var hostname: String = null
var cores: Int = 0
var appId: String = null
var workerUrl: Option[String] = None
val userClassPath = new mutable.ListBuffer[URL]() // 设置参数
var argv = args.toList while (!argv.isEmpty) {
argv match { case ("--driver-url") :: value :: tail =>
driverUrl = value
argv = tail case ("--executor-id") :: value :: tail =>
executorId = value
argv = tail case ("--hostname") :: value :: tail =>
hostname = value
argv = tail case ("--cores") :: value :: tail =>
cores = value.toInt
argv = tail case ("--app-id") :: value :: tail =>
appId = value
argv = tail case ("--worker-url") :: value :: tail =>
workerUrl = Some(value)
argv = tail case ("--user-class-path") :: value :: tail =>
userClassPath += new URL(value)
argv = tail case Nil => case tail => System.err.println(s"Unrecognized options: ${tail.mkString(" ")}")
printUsageAndExit()
}
} if (driverUrl == null || executorId == null || hostname == null || cores <= 0 ||
appId == null) {
printUsageAndExit()
} //调用run方法
run(driverUrl, executorId, hostname, cores, appId, workerUrl, userClassPath) System.exit(0)
}CoarseGrainedExecutorBackend.run
private def run(
driverUrl: String,
executorId: String,
hostname: String,
cores: Int,
appId: String,
workerUrl: Option[String],
userClassPath: Seq[URL]) { Utils.initDaemon(log) SparkHadoopUtil.get.runAsSparkUser { () => Utils.checkHost(hostname) val executorConf = new SparkConf
val port = executorConf.getInt("spark.executor.port", 0) val fetcher = RpcEnv.create( "driverPropsFetcher",
hostname,
port,
executorConf, new SecurityManager(executorConf),
clientMode = true) val driver = fetcher.setupEndpointRefByURI(driverUrl) // 给driver发送RetrieveSparkAppConfig信号，
// 并根据返回的信息创建属性
val cfg = driver.askWithRetry[SparkAppConfig](RetrieveSparkAppConfig) val props = cfg.sparkProperties ++ Seq[(String, String)](("spark.app.id", appId))
fetcher.shutdown() // 根据这些属性来创建SparkEnv
val driverConf = new SparkConf() for ((key, value) <- props) { if (SparkConf.isExecutorStartupConf(key)) {
driverConf.setIfMissing(key, value)
} else {
driverConf.set(key, value)
}
} if (driverConf.contains("spark.yarn.credentials.file")) {
logInfo("Will periodically update credentials from: " +
driverConf.get("spark.yarn.credentials.file")) SparkHadoopUtil.get.startCredentialUpdater(driverConf)
} val env = SparkEnv.createExecutorEnv(
driverConf, executorId, hostname, port, cores, cfg.ioEncryptionKey, isLocal = false)
// 创建CoarseGrainedExecutorBackend Endpoint
env.rpcEnv.setupEndpoint("Executor", new CoarseGrainedExecutorBackend(
env.rpcEnv, driverUrl, executorId, hostname, cores, userClassPath, env)) // 创建WorkerWatcher Endpoint
// 用来给worker发送心跳，
// 告诉worker 这个进程还活着
workerUrl.foreach { url =>
env.rpcEnv.setupEndpoint("WorkerWatcher", new WorkerWatcher(env.rpcEnv, url))
}
env.rpcEnv.awaitTermination() SparkHadoopUtil.get.stopCredentialUpdater()
}
}CoarseGrainedExecutorBackend.onStart
new CoarseGrainedExecutorBackend 会调用CoarseGrainedExecutorBackend.onStart：
override def onStart() {
logInfo("Connecting to driver: " + driverUrl)
rpcEnv.asyncSetupEndpointRefByURI(driverUrl).flatMap { ref => //向driver端发送RegisterExecutor信号
driver = Some(ref)
ref.ask[Boolean](RegisterExecutor(executorId, self, hostname, cores, extractLogUrls))
}(ThreadUtils.sameThread).onComplete { case Success(msg) => case Failure(e) =>
exitExecutor(1, s"Cannot register with driver: $driverUrl", e, notifyDriver = false)
}(ThreadUtils.sameThread)
}CoarseGrainedSchedulerBackend.DriverEndpoint.receiveAndReply
case RegisterExecutor(executorId, executorRef, hostname, cores, logUrls) => if (executorDataMap.contains(executorId)) {
executorRef.send(RegisterExecutorFailed("Duplicate executor ID: " + executorId))
context.reply(true)
} else { // 设置executor信息
val executorAddress = if (executorRef.address != null) {
executorRef.address
} else {
context.senderAddress
}
logInfo(s"Registered executor $executorRef ($executorAddress) with ID $executorId")
addressToExecutorId(executorAddress) = executorId
totalCoreCount.addAndGet(cores)
totalRegisteredExecutors.addAndGet(1) val data = new ExecutorData(executorRef, executorRef.address, hostname,
cores, cores, logUrls) CoarseGrainedSchedulerBackend.this.synchronized {
executorDataMap.put(executorId, data) if (currentExecutorIdCounter < executorId.toInt) {
currentExecutorIdCounter = executorId.toInt
} if (numPendingExecutors > 0) {
numPendingExecutors -= 1
logDebug(s"Decremented number of pending executors ($numPendingExecutors left)")
}
} //向executor端发送RegisteredExecutor信号
executorRef.send(RegisteredExecutor)
context.reply(true)
listenerBus.post( SparkListenerExecutorAdded(System.currentTimeMillis(), executorId, data))
makeOffers()
}makeOffers()所做的逻辑，在《深入理解Spark 2.1 Core （三）：任务调度器的原理与源码分析 》里已经讲解过。主要是调度任务，并想executor发送任务。
CoarseGrainedExecutorBackend.receive
CoarseGrainedExecutorBackend接收到来自driver的RegisteredExecutor信号后：
case RegisteredExecutor =>
logInfo("Successfully registered with driver") try { //创建executor
executor = new Executor(executorId, hostname, env, userClassPath, isLocal = false)
} catch { case NonFatal(e) =>
exitExecutor(1, "Unable to create executor due to " + e.getMessage, e)
}至此，Executor就成功的启动了！
作者：小爷Souljoy
链接：https://www.jianshu.com/p/9d97659147a6
点击查看更多内容
为 TA 点赞
评论
共同学习，写下你的评论
评论加载中...
作者其他优质文章
正在加载中
感谢您的支持，我会继续努力的～
扫码打赏，你说多少就多少
赞赏金额会直接到老师账户
支付方式
打开微信扫一扫，即可进行扫码打赏哦