Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
S
spark
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
cs525-sp18-g07
spark
Commits
5fde4566
Commit
5fde4566
authored
11 years ago
by
Tathagata Das
Browse files
Options
Downloads
Patches
Plain Diff
Added Apache boilerplate and class docs to PartitionerAwareUnionRDD.
parent
577c8cc8
No related branches found
No related tags found
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
core/src/main/scala/org/apache/spark/rdd/PartitionerAwareUnionRDD.scala
+33
-3
33 additions, 3 deletions
...scala/org/apache/spark/rdd/PartitionerAwareUnionRDD.scala
with
33 additions
and
3 deletions
core/src/main/scala/org/apache/spark/rdd/PartitionerAwareUnionRDD.scala
+
33
−
3
View file @
5fde4566
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package
org.apache.spark.rdd
package
org.apache.spark.rdd
import
org.apache.spark.
{
TaskContext
,
OneToOneDependency
,
SparkContext
,
Partition
}
import
scala.reflect.ClassTag
import
scala.reflect.ClassTag
import
java.io.
{
ObjectOutputStream
,
IOException
}
import
java.io.
{
ObjectOutputStream
,
IOException
}
import
org.apache.spark.
{
TaskContext
,
OneToOneDependency
,
SparkContext
,
Partition
}
/**
* Class representing partitions of PartitionerAwareUnionRDD, which maintains the list of corresponding partitions
* of parent RDDs.
*/
private
[
spark
]
private
[
spark
]
class
PartitionerAwareUnionRDDPartition
(
class
PartitionerAwareUnionRDDPartition
(
@transient
val
rdds
:
Seq
[
RDD
[
_
]],
@transient
val
rdds
:
Seq
[
RDD
[
_
]],
...
@@ -22,6 +44,14 @@ class PartitionerAwareUnionRDDPartition(
...
@@ -22,6 +44,14 @@ class PartitionerAwareUnionRDDPartition(
}
}
}
}
/**
* Class representing an RDD that can take multiple RDDs partitioned by the same partitioner and
* unify them into a single RDD while preserving the partitioner. So m RDDs with p partitions each
* will be unified to a single RDD with p partitions and the same partitioner. The preferred
* location for each partition of the unified RDD will be the most common preferred location
* of the corresponding partitions of the parent RDDs. For example, location of partition 0
* of the unified RDD will be where most of partition 0 of the parent RDDs are located.
*/
private
[
spark
]
private
[
spark
]
class
PartitionerAwareUnionRDD
[
T:
ClassTag
](
class
PartitionerAwareUnionRDD
[
T:
ClassTag
](
sc
:
SparkContext
,
sc
:
SparkContext
,
...
@@ -54,7 +84,7 @@ class PartitionerAwareUnionRDD[T: ClassTag](
...
@@ -54,7 +84,7 @@ class PartitionerAwareUnionRDD[T: ClassTag](
val
location
=
if
(
locations
.
isEmpty
)
{
val
location
=
if
(
locations
.
isEmpty
)
{
None
None
}
else
{
}
else
{
// Find the location
where
maximum number of parent partitions prefer
// Find the location
that
maximum number of parent partitions prefer
Some
(
locations
.
groupBy
(
x
=>
x
).
maxBy
(
_
.
_2
.
length
).
_1
)
Some
(
locations
.
groupBy
(
x
=>
x
).
maxBy
(
_
.
_2
.
length
).
_1
)
}
}
logDebug
(
"Selected location for "
+
this
+
", partition "
+
s
.
index
+
" = "
+
location
)
logDebug
(
"Selected location for "
+
this
+
", partition "
+
s
.
index
+
" = "
+
location
)
...
@@ -73,7 +103,7 @@ class PartitionerAwareUnionRDD[T: ClassTag](
...
@@ -73,7 +103,7 @@ class PartitionerAwareUnionRDD[T: ClassTag](
rdds
=
null
rdds
=
null
}
}
//
g
et
s
the *current* preferred locations from the DAGScheduler (as opposed to the static ones)
//
G
et the *current* preferred locations from the DAGScheduler (as opposed to the static ones)
private
def
currPrefLocs
(
rdd
:
RDD
[
_
],
part
:
Partition
)
:
Seq
[
String
]
=
{
private
def
currPrefLocs
(
rdd
:
RDD
[
_
],
part
:
Partition
)
:
Seq
[
String
]
=
{
rdd
.
context
.
getPreferredLocs
(
rdd
,
part
.
index
).
map
(
tl
=>
tl
.
host
)
rdd
.
context
.
getPreferredLocs
(
rdd
,
part
.
index
).
map
(
tl
=>
tl
.
host
)
}
}
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment