From f3a964848dd2ba65491f3eea8a54439069aa1b29 Mon Sep 17 00:00:00 2001
From: Matei Zaharia <matei@eecs.berkeley.edu>
Date: Fri, 30 Aug 2013 12:38:23 -0700
Subject: [PATCH] More doc improvements + better warnings when you haven't
built Spark
---
docs/_layouts/global.html | 15 +++++++---
docs/img/incubator-logo.png | Bin 0 -> 11651 bytes
docs/index.md | 53 ++++++++++++++++++++++--------------
docs/running-on-yarn.md | 39 ++++++++------------------
pyspark | 2 +-
run-example | 2 +-
spark-class | 4 +--
7 files changed, 58 insertions(+), 57 deletions(-)
create mode 100644 docs/img/incubator-logo.png
diff --git a/docs/_layouts/global.html b/docs/_layouts/global.html
index a014554462..91a4a2eaee 100755
--- a/docs/_layouts/global.html
+++ b/docs/_layouts/global.html
@@ -66,6 +66,7 @@
<li><a href="python-programming-guide.html">Spark in Python</a></li>
<li class="divider"></li>
<li><a href="streaming-programming-guide.html">Spark Streaming</a></li>
+ <li><a href="mllib-programming-guide.html">MLlib (Machine Learning)</a></li>
<li><a href="bagel-programming-guide.html">Bagel (Pregel on Spark)</a></li>
</ul>
</li>
@@ -77,8 +78,8 @@
<li><a href="api/pyspark/index.html">Spark Core for Python</a></li>
<li class="divider"></li>
<li><a href="api/streaming/index.html">Spark Streaming</a></li>
- <li><a href="api/bagel/index.html">Bagel (Pregel on Spark)</a></li>
<li><a href="api/mllib/index.html">MLlib (Machine Learning)</a></li>
+ <li><a href="api/bagel/index.html">Bagel (Pregel on Spark)</a></li>
</ul>
</li>
@@ -140,9 +141,15 @@
<hr>-->
- <!--<footer>
- <p></p>
- </footer>-->
+ <footer>
+ <hr>
+ <p style="text-align: center; veritcal-align: middle; color: #999;">
+ Apache Spark is an effort undergoing incubation at the Apache Software Foundation.
+ <a href="http://incubator.apache.org">
+ <img style="margin-left: 20px;" src="img/incubator-logo.png" />
+ </a>
+ </p>
+ </footer>
</div> <!-- /container -->
diff --git a/docs/img/incubator-logo.png b/docs/img/incubator-logo.png
new file mode 100644
index 0000000000000000000000000000000000000000..33ca7f62272560f3da274086a7532bcdec7b76ac
GIT binary patch
literal 11651
zcmZX41yCJrvgpCx-QnQw?(Xg$4iMZO5`x3Q-Q8US!QC~uyL)g55N`gx_w8=go2qZ7
z=WFk&>FVwou$qc23L+sQ002Odmy^=?d!6{ZnBZamf!dW2006``8%aquc}YofHCLz4
zHujbPfE-Fqs+ShlD&EMpl5Y`Ayvj)z5;VHp2?TaAVj*Utm^FJCG)4nO6k~m1gj`cW
zlm;{-c~oQsNV>sn`!CSWf@@#?MpHj1+;g}7;qNr<cjvS1bv*sm>nO(!g8bSXDU_fd
z6OaMfKnMI8-a^ij8|DY?JX(TW-<vTthRHE7AlcW`g60LFGVHXWynCqo{NX2_F)^_R
z0EALb$Zs&I(DaYQoAhHu4g&J=BOE)8n3CD?%(O8u$@}>^R5Fe*322q?uIRJM;Y|l&
zZ-Qx?RtvEKa+?t<6Gz45{&E^2moOVz5C98{<KCzNE6VP%M*M0{&a`npG_$+nZ_n~C
za80Z7u6g#Jk=bQWK|<ip6;^J=^ga`I?w4<<83dSArkcuV1Y`QA2^6EoIAi*#;g@9m
z17AoLAE!#)3sz`BVVXnL<pf8HL4{U&iKJ86yZ0p&u72Wko%IAcNrzDf{%ck&q70s4
zZ9<|S1h~vG@#|KWvf~d%-PfH2SzMm=Ov{=L4bt%MQ=QRbFMgqW&OX^e(LT3bR8f!z
zJe}2KuQ+!_L7(3uq}5%s4i*4Q(bIS^V;*M13!=g}!E%@sGf=AN%rg{em$Wm?zo7n*
zW4Bq86Y@d$?Er`eJBb^ZythmGaS9i(yVHlIQ~)!X!<f$YH8!!2s48NfxfVQt&JIBj
z6>^U^v>ID(z@MDaAFuD>M`45O_Myd8ui>Ao;RAwr0LEVYnCg$;MZaH!K}h9CSZH9K
zKu~@{wGW_lh9C_Hlp;dz7sHN#ih_XqB~F7Lcu0;`4HcS?Zx6dJ#=Qdb7zDLL&Ih?B
zj>iWz9Xzok(27C|pvQ+`?ZbZt(2&FTi(^M1+DQD4N3ek5mvA3Nx($~ofTu%M2+<&i
zpbEUre^6uBg4LGv%ZEK-bB7dwG7f>w_xp*?h1eC^-Ak8_T-D!RPj3%5Z$?85Dccvd
zgY8X_jfB^mwWH9A>KL}Z!}-hdnTILVw11y209de~4ogRbO>!88C^=M=s197P)xxNR
z)DE{5_bB|W#;dNez_?(=g{v$*|C7fW%^h8Ua7vt2vQ8!~ziJllr^AWKz4AS}FY-Hy
zWZ%iBYPKtU+|Lj+Fipgo5f!})=CS6}W{&1EY-II%)dEY%=aDG=!)EjKj1AN)Br7`h
zyq*BTXp)g#=S)7vY+4rNkzue)Xe)ye=M~o#tRwUjf@=TyKFJNlvv(&TFRreacTaWK
zX(w-&7G41wD@c$cx|h0|JQ?#5LjYkJo-zcXkYhIbL_Qi=im{Br2xcC18w?-xqQV@D
zKaj~(RiLh+)=orCV2KZt%Nt{`P;6x74`iZn0$$U4C0!?MtJ7g?A_ilsVrcTH6Vh4H
z$I<6umm{_yGQrgL6NVCo@`Ms0*5M#T(nKl^4Mny^;zY{g)X{fq0O>LqUKkoubQl-u
za?}X52z5&t1nI)*tr*hico|H!{A+_W*|l$UcPp{WFSPlqIFzK-W6Ha=P(6e_$R3D2
zq;RTiOUE^oHJNpU%h&1zN=plQ44Gw^Wou+Ei!-%IN*pVvHD=UUwM7g4W#1>5HuWd<
zJN1dInkU=Y&x^uZU?qdg)Qe{;T@3f5Xq4woIgQ<-K6e~ZGJj7kS^83QXdu`;ZPPWU
zv-tS)$QsXvY__`&>Rj(k>ZE#MwaQJ}%1Bq0S53Fvu*6U2DJ8d^RFYM>&geRMt6W5T
zTTxWI$LQnxAEZpe82O2O?Kxi?b65dbaahb?Vc2(Atgz2vvtch`HDPhl5i;&Y7DcW_
zq($uJU-#RFeTJ2XpOoQg+_AdHJd;F{;z~c4Hk1-6tt;nc*`zm2x=v+Il4fd7c1@*a
zEN5&t_8R<To@4gZGx`z2oWuN*%ALAg{fQB;N~?-Q2Wko5j@K^GBj+I>Wi`wu$zk4t
z)1u0e$TrJq-SKs8dWB`nG0He^)638575Vk+BO0+5aS+KZi2%u#fPsK3u^q7!zj`)g
z_7a~5Usr|$cNVYT@kPh9)^Xc3QMO-pPg`qygpq-fjB$)Hma&~N+-c>#voFGX5Yh0|
zh(nBF+Pak&o>#*w##4`wz0jO6p>XaDd)7c!lhbs4Spn_3TU^;f)KSVw^Sa`h!_L%W
z?VW!=8_1d9A=*j_=`&$d#>?8smh6Gs#&J<y$2G&YqiC{ezG;GKPfz5x@86m~_5lL{
z$pNJS4PwW=DS?~f>G|>b%;LdJ#<<~&y@W`fC?+@?L=vcA<O)<DOj`m5#LH}?8I;@(
zkE;P6nmg#bwE;1FMti%JBIl-Gmb*PuBhzSkrko!(A(B!iQl?bguJRV<2jO&i<#Xk5
zvq!UuC%O1T=^^Q1>2VGI4VDdaFrG*{%&o*_IBA5-7;#Z?;JP?6(~}<O_5K9t<B<Lk
z!Kg{xL#FmtJv*1-y(C5KwE8q+<w2@p8Xg*d%kYNtm3{f@;<aM-SXL{+W9qa%=Hu8~
zAs@?5tFhcM%dr_vA<dT3=-aLroHyNy?ux|<r@5dx&yuf7Xo}l;A{iK2vzg!-rAZ+Q
zHH=~;-?TI4DSZL*@0hrVB@-KI4u(85h!~4j;`&jI0)v%uyROn8mKS-B<jWM}q<0&i
z$=f3SasHreMn)!Gd!rjBUh=2ql#`U~6k#7&pYz)=9d_MIeUus%gUzO4YvsPogyujy
zv}(#)MN=>CZIK?PkN4jz4>>UHFm^~`-w@v_D%1+pwD?;4%m-@oW<|^XZ1l|9m@f`r
zOK%h^4m}Rx44EqOQK$L4JeS<2jVX_1{Z=$q@Nyh>#I^5k8uFV6<Tz~HXxy!F`_Ymz
zKOQx{?PS<^2^zYHG+qa<Q?Hk#6r_-74{K*tb?e+!IW0dgd|x_SaIAiB4er<<RvDg7
zHE!>S^!mD*)Bbvu+bitgQR;!?*WlOmB(jOuRClfWyAI{S;9=-t`w{6;5lxEt#KGCY
z+Iqo;$ss6<O1WqXVQLxeH_3%{gU#xE<(9!0c;ZB2Ul+ccUpbT%vc9o%aV^B0{5pQw
zzMbVLwuZAjOVdtio`W99+3+H>^QW^uZ@lZ<k6r}si`iO&`o59gl{*UuryKM8oBdb1
zRvOdDPfVvi&Ypd%hFy1luQQ?99nK;<Z>u7ow@;CB@!DD1O@&;*{uS?y6NfE~)4H*y
z9^Jm3+a*1hndi;1=Z?#Qa7W15?Ym$1CzX4$j-47Djk|@fAA5671ij#}U`xE(yz;gu
zTAz;mAK$@nc4Qz@Twx7gtsA{#m(6S3o*xRv#kApXVYNcwFSno4Ip1#MsztkFv4u*6
zYP?H7s(0J=?`mfQX3tr1yYYSKzwqBw{!%>@wKL7?5qZShuVXYv8kX6L*i!0d_9=UD
z{CHix_8T4;A^SG^#&*+il5|uCm3Nx=+CAv+!_)A_cx89U8|9R<E!i!aJ(-=2-6(xg
zVE_n?c6Q|eI6Xm$fuRC?NdSfY?p=bG5X#FZ@RHxWGTuyujIkzg;|U>&jgaCYApysZ
z08~x@oA?pps1Wfk9c(~O_5q2yB{Spdm<uAghM-2KmXb=hq^(po3}73UmX>ynmbPb+
z(&ObTpj}jHd6mPi<TC(Yh-Q=hzVdFlV&sGW4K6J0F4Pu=Nb+wji{LD$?*;%MVg0ip
z0GZkN002a`jh3Ffp0bjFxswBnnS~R`lEvG>`7bp9AmlCZ7j&?6HzW6Uuy=G5@D>LC
zi$dTp{LeNkko;dH?smdJJ!Lg=NheoJa&8uO77m~YA~`v^kgLUK0Szgc|6~7qCk(W9
zcXt+GW%cs%V)5c+adNd{W#{MTXJzAH<=|lcOTp~s<LGYY&Ftt#`M*N`pByPmH*;4T
zXLlPXNAiE<nt_}=+=YR_e-!=u_+L9My>0%Z$<giq-1_Sv>pvJ)b{00)f6M;MD)i5;
zfSQfBrM;e%jf175+g}|b9PB(o|Dyl@F#pl`U!3~?;pF1`Z^r*Z{2!wb>p%AYmp}g-
ztbgtPjg|<a5bM7qFM`N3ceD)vkdMksiEDX7TpGir8)>ilmt9;;UR+E+=Gx2ibV1Cq
zb0x4y7Y9S*NWnZ{q{ynH6P-~$=NAeENYN``1j9pMzz6=sQt(|@=2>sD{QP>A{QB*>
z&5y^+%8J*pvH0zk+G~dSINR^K?O51U*vdW(eOSJ$OW4}MLEO&{J;55Er-Ov2V&2RP
z$SsUSPhYX$Ibd-ERRCjPXlRz^bbj};C!%6$N$q*L7OLO$x$pXE;M9G1%Bw#2dkh?t
zu<&MiMJ@WLm0jiAzC%|<!4-5wucD4BBw^u353jz<N1$LC`WXG6;N~2fypB4UtL|4v
z0(?W4O^;;?R#qpm4xz^tb2$|i8DnD-72yXhsul;?!*Es#J<}-ebHW5;4k)}0Z^pSf
zPV5s{Y!P_WVdzvjs-X5YW5^JdVyh$azSAIzk9GmUVu~#EGXxCI05;}ZW+-)e6l|3Y
z*l)JK#eXvG`whm4{=v6fs#Yi2JFBWAHMNBMWD~U4vhFxwCHksw^zHpYul`e@?|9Gq
zr4u|f<jK|5@DQG9^qL`yq+LA`i7~~bit6A|Ejh<{;!h0akdAU>@M`EUt5<dR{s%dQ
zX+FTV`wnFcArp3+r<T2pj7;qwO$cP09C<wvMH>wVzCCngJm#+hRW6243Cs$;O3XO8
zT}}{9<|C-Yo-1aE@txNbtUb|`v3x-tuc7|G=kxYPME|g3Vq*SWU+*x@H7j&#cRN<u
z_;NSp{kUr7GR=QXr;?PK8rJjie*Mc^h|l)75tZ-@lR-xj-EDhAQw?EMW1-%1E0IZ^
zq|`mUtDAATs~4q*35TbnTfY7D*r@Rr1lQ7;yps_!&G#Zq20_7fb?r737)^x|%2xCk
zsaZ^G8M}N0&u~c@jWWnc<rPGJl(s0y==_l#aW3HDj{)I<;6bUGbW%PO3gN<2yKnF3
zrcaD|_in`FHtOmaxnFMOh2HPyTR;B1y8N|wKixdGuTVOw-s4<naw-dqM&!ENB>0oS
z#mqp1N5pz9hS-UsU9x8MNxI(B5a#IEr{U<vlWE+!$aS8B>)nZ9>$Z%%#GYqdr2`Sh
z4UdcsxE{f;XlExTRi}WRmQ1e}B1yrupj%*gkJ%9;h7sGZ+=TOpEgrc?cfbIdX3fV0
z8xXN4H$>g^sBfAn@>QV42@kz}QvrH(a#BiJIfiicK{=JhxKb*_=kAC?P>^UMm07d#
zsqHv73QJW;-D=?PIk4GonV~JK&|F*`Vw!Isnb_~5;oYYdEAi>uSw-2eSy{3Q-%^#F
zN*TQ+r$Z)eMffdd!<L=<dDRh{vK-iOWTs4SG|$$jEkCpluWMLA3h5tvd!U@de7RLr
zG@)OBKR0pT=up3H%+eOPDDt@4ixjsN@(FN~)fhK7q7mn2hA{^!^9fDR5SI~*A((s*
zTiaDV5mPKjIn8CulrU?}R_9X48)IY)IzMr%-w8SoVf1-2^oxdPI-pzZ7*`kg8Vd>K
zk&;=N6M?VJCw0*nwI~@VZ8Cs}>j|B?Mi(Yc2_0E{dus+UxlnitqE;k&it4!;CVIXv
z6RkNbxP7^F`eu0fac=t2|8ktSZQ28ioiY~|2CH@rylErzIlwUTT(vw~Idr8bFNc~S
z8$bE6?cX6F*>Ih#sUj{d9ZC%+RXr;0;NT#tr}vahuPK%wjWwaJL<JKp4nqnV3MgEW
z@yP>C-X2cC1?Z)TqR)G6f5n>-e#*9utE^;jt~coTOv~f&4e!M#yol{9#T;>)9GGuc
zmNha-BmVrNH)zuR%Yspqw2gkb^=6>GuGDW;^uSzC?3as`4PZ7Me-Nn`oPZ|Kc`QGh
zgBneHPDC?~9|AmWGn&Z(r>QXM&p+#6zP*kjQDCBVwGo*(w>sBA4B9;tU#vyMqMIWi
zz6~T%YU8)qwpk^H1P001(zD0>j+T6+rcPVK8&uC4#CXZtexp%PR9tKjsY7M@_Dn%9
z^7i+6Y7vKtmHHQe^SzIAynZe$grJGOq05r`6t~}pi@u3l^--r86$;4s`U?2pWo^@`
zNU2jXGFFMAdFk;EVa2w}l70Q&{y5H97eJ#khvxqaYDj%F##-Fg-d<~?Owqmf`pdB(
z!%X0xaH!!dr|t?^d4qmFJ|n+fOkyih#A6)eC>w$i;LmOxMxN~$Gyz?Zax3b@8N)A?
z4zf7y=KallBD70uu{6oSmH;%5u9CTMT#)3s*0a!^W&8u>MnY7f7=+rP)Owh$_tpx;
z{#N+xpHY$(-|Z6b_di(m7vrkCIYcUhMZ7uS%X5<$!JU{N%;Agr&kaAEj9HC7g}Hln
z2#Lk5%>I}VMu7;Yvw?;U6f+*8fVH3w0z1yh!g?JFgk*_f4ydS@hnqze>P}AQkX(M1
zfiwoW(+lDZ(O?R)wXKz;nM9ht?cv!Ln;%p1RS&B8^}*nkYo0IHjdVS2MLDdu&|@%j
zbK~%Cx+f7Xjyargl3U9u(fe}q@&;4VgZD5~j4%biR3AoV2jx>WTt%_8%U$Z~>IQ?>
zIcJ25V#nnDADi+j-lwNkQZ%qFUfuHf<G~JTNl8VyNr)nF=-ZWxe9V-Qx2UYd-9lRH
z^J=H*aio5%wnZ^33(pKKKRyFR-9!?x6wQ#pEQZ_grc?wiEj*JdV={5~85M=+2i|z(
z)D#$peFl^0if6;0p_<-cJ-K*6l@X4YC_LYLt|ba<@EmGlMyH(f-%^Phm?5m=Mf8Eb
z*odFaX4{t0PRR&uUQv-!a|Bz+c})+7q|)>}py>*;iGULPFvNs(N|&DMrw#|eAXFIq
zwyuaz>XK3<7>hgD1j0;W7`@U(3wFg&qqxXCF-S}X*E<v4k85@c7nOB&y4TaO$`g|@
zyj$h5`|lLIb5aQre}ZSe9C0d$m)L)K-VJB$37j5C&XLrt9*O~Os978eV5{0tQ(DVa
zC;g5r{hPO%6?c#i0Si7)UJ4}9Gc$pq!ICmEum&dnh!xX1$uAtcimp4O$=LR`$`XoY
z?gzP;#3t<khr*tF1vpd94&>%O1?+<v8J#s^by!Upbi)MuyL~o`{pO=h$XF^1{<6Iu
z4+p*U+!zlCjLkX4hT0Mc5i6GVpAjH0tnqc=C|cVrhGSsS52$%8-=B}aiAMQBD7mJ$
z0YZ6rW8D%K(}oclijHBl{FYi%yi|Gy&~4Cx%jfC}ziV?2BSIvdBXK4v1R^mzu+`{-
z5Uk8T@G>-#D99U#Tuo$TS0wwjiP2%?As7@`u$w7X=0b!Z_VvaXY}MT(sZGdGnHY6z
zwWL=l_~$-u(|WYT<P{V&K)8Z@{4>ive;)o+p5`pw|E(_e{=T&f;R>A~;hWLQi5r2y
zp-${E>&GSEXqNT4IoDn&X;H?9KAZVc`dY>MWYaF_z79u8ke_hr@3o&R84;b(ysk+(
z>6zKFMy7}@7grMPfWpCyz_4Uo?>P<iWzFy$;cJlFp#W9;oLbg9y*K83J@?O8CfW9T
zNmd=6<^AVuVD(Bn5~&r!h76JlYdRS3oZ+UTx?*KfKfJ#f^gT2PUOH3#<ZN&)ja&xW
z5w+OEJ--n|P`UlW8aXXUmYWf^`|DXvi^uf$HED%hPZoC!5-vjz_oAShk$@r}M14nr
z0{D{(0e|uzio(D#btPfxD8!|#*(z0brN{wN7kE8ueq2<ljUgJiPL;3L)rI&q#;jdb
zHRMvphnaS@>aTBiX+3W+|5Ppwz7Mu)3z>pmjvw3t$Yj1J6mQ3(AALxyo%z$6U-p{j
zGiCN(>+~8VkkRSiLAmYZ-&P2wh3=H7D`)v-)xjGPOXOz~4GKLhc&zVpY2C%yehcO&
zgXY4-ITcW9W=DdgOoNZ+>+uSz{A3mL$Gb6ttP-mA5Cx76_G7*4SRC<Z(~^!7(8lt@
zkjW-eC2#_XbGx|aGpV?=3=$c99<%%_i}O@V^^RPRI{W+Wy1!I3N#^Dwcb=3n*ir@<
z^Ut@8!-ZrCP3+cUZ7QgrKiT1o&)qgOS7BAK)J3?>H&i#m`g;}1Ns8RdtPQ!C1ZL2)
z*jMB6jzY~)Y`#)4z)TViHC%o7Aa}Q=d6S)#Rq4Y?7x}sya1>{Uvfg-VUTkFnjz|uA
zY{}P)tBl`?9}g+wvy^gYkT!A0m9LoiS;z6yUMWtPnvF9?;TTC)G9tnn`$$9+LV*uc
z!qJ4CqsbLkXh9Cu|8Hrw|GNFrJ+2@&nL)|a8OP(W7P8u8RaJED|J3ox!~tbrFr-5A
zVRno*FZ$b`yS)6)soCtq?{p)bNO&w-`R1OHDi@R38MUo@k^c2(?W6S8FO8Y8&6>9l
z%cdiHLs8mG+EqHct^2W>VnsmJtCG9ZcDl$ISnu|Yz*I*Ae>Bf(Ux+D8+?37^l9w$l
zF77Bi(SyDXXd)gDYO_p10&VAr=L2zx6$o7o5uS*vy;PM0JR}1Ypfv`PzTbDetz-*#
zg3EF|EVteh2~8*Fv!DlSM@&W#h(|+s=ob}L-?fsol`6pI%Kl!VvvB)rDB;w95`%2o
z?_u3V!vwYE`x3?KZeJ%CS?2Kv+NR~yXx3ROgb;8qlc;N$X_Ehfr??cS>N5#+YHcm*
zMfFM{aU6C>B?BWgwkH-9(35|t|Il6D7B4@7MWR-MM2eklYWRS<d36QdSxU+a90o(h
zMT8{QKX)pj;ibpZBMPM%SsL@3sLt?*ihi86>td<79W@vIoj>wPQF>8d|Lj|p={p0D
z<3_m@oWLBHm4(ll0N47Qz>qU)q~d)-{7GDctE$1oGchzljdQ0;LTrGUtW_5w`2twv
zc?U`x&4a>~S(%Hs=WktIiR@9**vLWr?d>L9iyXLa`*4Q_tnuT<^n_8H%ZUif)Pe3m
zp1h+i5=xGUphO4-w(Anj&7$yPUH~hBAvuugr2Kg-AcciXya4wiK7OQ+DztKO{^flM
zA9E=CX{lMsLH#RCHcxQT2F-SD5k#LHZRUDbNK*vEjyvU879(t6d3r9V`iWC70iF~3
z=7yL|yW+TX!cJnJX+ZXlYd$CqLUkO>=XeuWE)x%TQGjy3jdw0nfEZa!hl@C2tx%h|
zldPUnal~}L6vZZ8Z{2$zC=DU)j(Zt@;35Sj&4)NtSI?Itkziuh3{GeFBkx$h<9c;e
z4L+PO{nL2McioXmYe(pS)cLkIztZ`I-m`i$!mY8t+!mcEekJJWfW}-?8*eZiJU5A~
z<+V1XjiFH~26hh3OeKYuc4Sn-!cYncwpT@Lfqe&_g|iLW;q`t8^Cg88PFgIuhsCoJ
z+av|drk+UZCjWND`%LS1V_Wj<j+vw;dBfIB!pX8tg{t}6TT0d|%U4k)6)Er&tM7n1
z0f0pAc4i2WQ=(|mMd}hcON(jmpo1|04aXl5EP|zefPmUTcY;Wl7>CP7l=38!U%%U`
zYAl;y9w3F3+3wg#wu%QM^GDJ>y*5{6D75-ee}sypo!R;96fcOV&D&0qQi!Y@!pjgP
zwdWafZJ<LPSbDGHB8QkK8xwFcYVpa>kg{<ui8`?jD~6<5?t!MDZ6XZw^sPCEzZ{2G
zO5Yc)g-b?lc4w%+-$Dyopq`FC<o3$K!dp}70Ml`)0jCTH%7Yn**^aCg>C^EJ&KPZX
zhe{k&9#-p)$)a3YOs8Xy<5^gMjZa$*4(ca?;-46PsFL=a!PImCR#q`+!C6VM<M<6}
zOHomzvG+;l$5&aaPfAgVHXJc&`7hEKg>JppJ7ZD@sas)SRR4&Zm^5|1r=WJNN1?I#
z!rUttaGd7{JCSF|;u`^%Hg&T*mSQ)k22eUj0WDsU?h+yJ(Mv;9V*JPy5FT|6-AS0r
zOALK<#M4<?KuX7K;F}5mjY^x#QrF+%U<~R##Zh#CrbZymHcn>L$s)EmRF-6`f~*Y!
zbdpwLfP$`6FdH1gieFlH!-yUOLJMa{$hu<aHOnZ;KbwcOQlS&~>o3S0<^t1Cn7K$8
zXFB(I7W9=If!|V)<%#e^1sNXIcNxC3NK20a#U#2)T~%|WgDs~wD5>h_zDvcFfGc)1
zugb0x>BRDiKggKHir|^SZ-3}p*mV5sQHhqFdd*5g;khq{^_wPs8M*NSK?#PgllDR#
zRVa2pE~RwDyE*p0oEvUyq$&$!w^73eojmyLp!a}G{$Q6}8yGg4T2-W?r%q4nP*J}~
zw{4a%nY@3R&BtJo#83{D=8KdBn`8GW^_EB4z+lx&Ny`-1g|;hvXXWmb_&TKZi(BvW
zv;ozhY^rm&80#KM`=e}&;;&56-X+(;Tep&-*2Vb^z|_yv+5FbG+V;6$zbX~MF~>AW
znM2ovgbO^JGj$NzTJV?$&S_h(70S1Y)+*|S-_B_+=2z&EaB}l%5_1l*Nd|^w(*QrQ
zdl2C$%Rh2PSR1mXe2AyXIPdL-lU>0fHqGz#v!ytF*(ai;`|4~_fD?0csRv>rjKRsH
znq_Ua7)uoYK^1|2>spSrXs>STn=H?ZiBMEjvj`2rYy^$W%kYRf4Pv+@r24ARJ!kj`
z=qr^l)vKz^EKm44Hlg{|LL%f_#dbt1O}5qFykh8t!Z4g(DW7Q{2)2e;EFwCqlzzJ{
z&?b&QAY`~O5$M>%hlUzr4l1jg&uaKY1FhtYu68sWUhHL3g6jy92)25Cj6kz&PR&=s
zxEz)+eeTDD#d$53T^C1V^Vy4cfLN>w?!!fR_*GD3+#2Z&o0KKTkYnt0@r1|*t(^BP
z{fWN2Piy*emmI{=(ufIJjD~M7f5rhFlYwvE62kuW?$z^35-Xt4-!~(usysYL@z6O#
zrQFinZ4Xh><n9UP4A0U1gMof=iiocudaAjDBd4?l3Myue+oPkw_1;p;p9Q59BT{{_
zX!1XV(q5qmXNQnPTGxwsGB?8iF^irjuBrj@ezOK2Ifo6_ld3!ebvMZdM<J#Mnds=$
z5@5}JU~%B+*sZgL1!hyFTJUOAzJm^rkK+zqQW%3Urn&am1oo4t1WogGk3!uiE@U--
zp6wGWJjH)?xJm4;5d$8}BVsAFQAP?0BA?F@OM4!syIoUjEA-gt@KE&IZ9p?M&hGN#
zw!hs@ED~x*Locl_cRw~$RDl50DV97O`>pr}{XTTjQ;ow>DB2J>My_m-(PUXU$$e=D
z&NHGPZ)!seOiI&D4UWDn6aQ3-5XT;*0vC@md9NGCTInc~_V@#jUDsdH956MTc|oK{
zmMbx_t)C6C@$mLj4IR~?#<tGtO{|{L^Ri&myrni1w<DiGRJ`jk;%Q~su92CpBy~y`
z{3_k%C+IySc}brdRMH}*L@7pqK?HR@kDWJ5l^=xHU0wap)>B!-Va?Y>hP-v$7C3lC
z?d|>6<#p_tgy%70`D500NQ5LLmoi?uWfa{6l0iV~N@*?k)dMIZjta$Ej|!nlB5RxE
zMvt%)=`cQln8;h@;nXQIqVwf$t1qp-DQnPd=0F6Qi;-Ne*tInNhv2is)tq77?%Nb`
z9}j}k+g;?074C8-)$c!k5MgEr+50T@`Ta6!pbDPMZ{!eHwtQWz^XEPxJWp(6I%`+@
z6y;}JZ8nTRo6UBT=`JN1fd2~rKEkoN_KEboW8^A|V5RlC`}Oz42?>FH7&EOuQjT_s
zJyOPz!=T@{b79N+1b-dj{Av;OrvM79*S#%W+~J?AA_)uAhfya_<n+^KMSsLy6k}g!
z)P>Fk%W9LF(hI|n*szrk5?9T&KXs3BA23h4DN&>8BhI+!Q@I%T3&VaO=6P_rUp6ma
zag#<**RcBEL<LtK`nhNwAaeLoX1dJ?oHIT6`%MU{FY^~%+~s`;ZTY`mivRGwpOtml
z4aIv@6(CA3xQ1>cT!x|?Awip6>G!~R?h2<=k=?n6t`$XB%oe$M-j5r}-C`Y4K-T1Z
zgNHo*DD|j9V?*Q$0))bshM>up%{`TTWQnnh_g2{x!C+poRZ(SgkA2xl4q{zS_wBk#
zgR~TW4LbR0PD_B`g2Lp_$N2vEqX$O1BmtAw1%vA~iB7{60`<2w7dv8Rj+YaO?y=SQ
zDGC!WE{F)3I(4WjPJQRcmhX9hV)WqAJs2wV;@%|!x}+710DHQ*poq<;q!wS6&c_}H
zC)8=g7AU^MHn##D^P5iNhmSIGG6QtjvSrf4@T69M9xXnT*WZzNz{uu0L>bU-Z;!Z$
zd8RSjs_1y`yj|opnepe_0j=n15A)N0%*H#x-o-DHzf()X<MCf*eB@g`$2kL5k!~qQ
zqi3}HRNYWuCH-;i*543Cy#+X5Of>PqFp;5(qeS#-<yv|%VnrW%5W!-YS3q58no{^u
zF&vza*pOrua(yDx?XWxP6$E?^Ohuzqx%nv8?((DN6Z0HgT-?3Cy;++9C1Y+yCN6_;
zlxA8+M6#w&MLFjN73nE=r}YhRIa-$U^Q0Q%*sq*D46EeW)K*dnAD@?ekdaWxhU7hk
zr5<=nEWRe5pfYoBvCr=Act#gC6x_0Pdi=8@2^3#ZgpT}`c(7kvDgWd`0Ef+a7a=fn
zPV(TpoIxY&F}E7FE{3&SGji#{SxRKqsL#+8G!NJ@PA|2yeeP?n#$)XAPGJP&dkBQL
z(fFu>Zz|QyP*m{Vi|G5Qs;#*2)jUvIJ@oF;Mtx7lZdEEPX|(~~8xxMk;DN_}B|r9o
z3^widD_yFKcBbB55D_u)^LyJLF&CGc{EA86)zX#5CG+9^hH9Qejp9hzY;lDN0<Zw$
zG5*bJ&0xh&<{mhJnwr`ckvH!piAGS?)A#4o#@fdIcHm6W4!Kpy`~=GbN|NdY1fqC8
zAEM?<I&=hNWHLJhmJTr8NNahXn#&gm{$y4bNiDo=QnqxiGtVe!$b<o%jZp#a-AVqO
znMj}{CmaZf(bY*KP##Cczb+$cm8zui%&w|!S0F=`JB?~MV_Axd4)~c9J2Sn=Zw~gy
z*bH03v8{f2o{EZ!T3oQt{ejB2^u-842Fofnv1%ysp=%G^n7vw=$KP@xMHUsD%h;+m
zhHDCj;plV`J;LGh4%)*pqsqzo7MfH~fx>W&NcZ(ssBAn5N}l~llQ`T`2^~X;AY(Q0
zytl-=FYt?h)to#WK^tNWRrZJ~8RU=<i%9`^gC-T;LgQk?FL$4J7GOr`G=yKPR3ykg
zp=J?H4H-~KD7){S&7z}q;5LF8n19WgT>dyf_D4;*X328wa4(^Gb99l{X-kOCP-u1~
z6MUX{$*Qr{`#qXzVOa;oAGT>^!e{d#LmDdVSc#(zVW!Cp0X0a5C(@K7F4XL|t*O`$
zwP~9V6>dz=x+^)>fb|}RG(u+PD#pj(A;XCF(Gc96X>xuP4xEjd`TGJR2%M8eVvHrb
zvvO;65|<sIqEnkf%vzotYY>!9B>QJYz`Bjni=wR9q=^?q0_{`{mG)Gcl%Oc%qMhUI
zOML!kHqIZqwsrs_6m{=!XK?BrQJ9sIlo*^K$<C2^DQ*+AoDsKNZAWo&*{#rbjD|9l
z7#1WZ78tM*DCGzWDndht4&CDu+J=koo{i{L&z72Y>x(R(v|387e+vR>ZwJ1G=g^tx
z!PZ!CMg5Z`T~GiQ7x(3MdqP=yh4$0bDSZ9Im^gZjC^*F9<q(-(`w*drFPi9?tqU4L
zdwWMlhRmrGgA<Ag@d5@yAhI3Ae5#=F-+u%|+vujAjT}bW%k#Tc)Z7x{Z_p_z%ZX;U
zhg!vhg0{Y)I)86TXf2%PZgj`h-5@}ruXj!b+5YaeR8dBi@W~_p1PYSU3xOdPa%i-H
zK@^9}iHU5wcfnzSpSC-oSzF#ql3vo#<dnI~ex+vh;OBX^23uAqEcnjeBrKU5FT`1%
z$So)e#(YP0rHnnuY^~+kUl~?3+VZ55nl6wYOE|E~6})PttR>}~yPX_ajl;w#Sc)`c
zfJH|!5aH!#)7>vnk%k(uu9$IWn&`t#t+X}|l7x_Q!9+oqm)0ie6=xzpg$u8D9WwT2
z1_jC0WU~hBCdxshJY+$>z|;qa(v>1}H;(wh4a*J1c3)ih$`^ZERv+svRl?s9dZfdV
zeNn5Sw4V}O5K*qYs9qvhXIWEb%*7d$Od-1xYO5hF2%i`^ECc044;&sRJG+j7fh`F8
zuw!j>7}24>`y{X&b0cKtX26Ke89Qv^{cecZ2V58T{rta&SX?no(!NCwz_S6vDyquO
zcNu7`fuooSac)+qG=;pG3Kz~KF(jE}!JORY;Bju<Itw_$*k>$)oAD;(Z)9P!@JkOF
z`-{?F1KQ#7Zlb7{W`1H20*57uyAV3N76(kJ@>jTxL|Rbrbj$GZXW-d$kE59<eiHm1
z0WFuuPVp`@%==Psa^h_kBD<JGwe58Qr1_Mz{O2ZV{Bb&hi6ue32QZ<tY81QmwE3%0
zR3^u)xSFEsf3VfBQo4Zw3aQirNQhB2KZut_1oN~pK3nSe<i$8~-<@UIRH0g)ubfLl
z1XXRgDKNitFb=xN=%<5V^8yi>_ea7s?0rl<K`ayFx6y5d{qn$3YY31Lq40+gX(V0b
z?9+fL>ZfYLCw(*&l!d61>|(UBadpr+Xk$g5Hm6uX5KpH}H3X4uJwX%?W1W<V>^h?y
zO0Bc=IM*-vV#caq8bi?@kB|Z%q0eSp>lZbfHdr1z;dx8PJfc|$4*pMSY|5VgMfGx<
zYRP8h>rZR3C}2-bF7A|5$*Z{iOF`@_UUmKF@~W!Dyi?S2>iBF|C^rgcmPG~?MZvEe
zN>g!_*x!b<zw4(@kh*|%^wT8*FmW-WCz@SvqHJyjgRz8t^|sA{Ps$0d_Dw=IK80f$
z3M|ZA*a^c5#N~S)N3#nfc~|(TpNcB%sC=G=IO7gPUIkQ*>VMHd=<kZ)TWgW^Qz(*W
zi0WDBGjDSMGC>5=Y#2SQp_azxr|5`Gkc6hL47Z_0#0+Aq%VcJa-N8jLVbDlqc{#%5
zj8L8vzemo>eo=LYM(Ga5T0lp&xX+dq0wk~7PS^<cN>pd~>T}}YM#?h3jK6O}4g^Vy
z665WhCv&&^T2yEY5S*UtDj;0~N)1?j5%`e;CEPeIw0qs3EtL&AA{>&qRw2VBkp6y1
zT*9MvZne;;<2wT0a$+v%TZ<KG9VO>+?yGu!o14vFI?me2``c#O27W+|2t}(axi1+1
Q^M{GNw2D-%#HXPD0lH(mhX4Qo
literal 0
HcmV?d00001
diff --git a/docs/index.md b/docs/index.md
index ec9c7dd4f3..5aa7f74059 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -3,13 +3,13 @@ layout: global
title: Spark Overview
---
-Spark is a MapReduce-like cluster computing framework designed for low-latency iterative jobs and interactive use from an interpreter.
-It provides clean, language-integrated APIs in [Scala](scala-programming-guide.html), [Java](java-programming-guide.html), and [Python](python-programming-guide.html), with a rich array of parallel operators.
+Apache Spark is a cluster computing engine that aims to make data analytics both easier and faster.
+It provides rich, language-integrated APIs in [Scala](scala-programming-guide.html), [Java](java-programming-guide.html), and [Python](python-programming-guide.html), and a powerful execution engine that supports general operator graphs.
Spark can run on the Apache Mesos cluster manager, Hadoop YARN, Amazon EC2, or without an independent resource manager ("standalone mode").
# Downloading
-Get Spark by visiting the [downloads page](http://spark-project.org/downloads.html) of the Spark website. This documentation is for Spark version {{site.SPARK_VERSION}}.
+Get Spark from the [downloads page](http://spark.incubator.apache.org/downloads.html) of the Apache Spark site. This documentation is for Spark version {{site.SPARK_VERSION}}.
# Building
@@ -42,11 +42,17 @@ Finally, Spark can be used interactively from a modified version of the Scala in
# A Note About Hadoop Versions
-Spark uses the Hadoop core library to talk to HDFS and other Hadoop-supported
+Spark uses the Hadoop-client library to talk to HDFS and other Hadoop-supported
storage systems. Because the HDFS protocol has changed in different versions of
-Hadoop, you must build Spark against the same version that your cluster runs.
-You can change the version by setting the `HADOOP_VERSION` variable at the top
-of `project/SparkBuild.scala`, then rebuilding Spark (`sbt/sbt clean compile`).
+Hadoop, you must build Spark against the same version that your cluster uses.
+You can do this by setting the `SPARK_HADOOP_VERSION` variable when compiling:
+
+ SPARK_HADOOP_VERSION=1.2.1 sbt/sbt assembly
+
+In addition, if you wish to run Spark on [YARN](running-on-yarn.md), you should also
+set `SPARK_YARN` to `true`:
+
+ SPARK_HADOOP_VERSION=2.0.5-alpha SPARK_YARN=true sbt/sbt assembly
# Where to Go from Here
@@ -54,15 +60,20 @@ of `project/SparkBuild.scala`, then rebuilding Spark (`sbt/sbt clean compile`).
* [Quick Start](quick-start.html): a quick introduction to the Spark API; start here!
* [Spark Programming Guide](scala-programming-guide.html): an overview of Spark concepts, and details on the Scala API
-* [Java Programming Guide](java-programming-guide.html): using Spark from Java
-* [Python Programming Guide](python-programming-guide.html): using Spark from Python
-* [Spark Streaming Guide](streaming-programming-guide.html): using the alpha release of Spark Streaming
+ * [Java Programming Guide](java-programming-guide.html): using Spark from Java
+ * [Python Programming Guide](python-programming-guide.html): using Spark from Python
+* [Spark Streaming](streaming-programming-guide.html): using the alpha release of Spark Streaming
+* [MLlib (Machine Learning)](mllib-programming-guide.html): Spark's built-in machine learning library
+* [Bagel (Pregel on Spark)](bagel-programming-guide.html): simple graph processing model
**API Docs:**
-* [Spark Java/Scala (Scaladoc)](api/core/index.html)
-* [Spark Python (Epydoc)](api/pyspark/index.html)
-* [Spark Streaming Java/Scala (Scaladoc)](api/streaming/index.html)
+* [Spark for Java/Scala (Scaladoc)](api/core/index.html)
+* [Spark for Python (Epydoc)](api/pyspark/index.html)
+* [Spark Streaming for Java/Scala (Scaladoc)](api/streaming/index.html)
+* [MLlib (Machine Learning) for Java/Scala (Scaladoc)](api/mllib/index.html)
+* [Bagel (Pregel on Spark) for Scala (Scaladoc)](api/bagel/index.html)
+
**Deployment guides:**
@@ -74,27 +85,27 @@ of `project/SparkBuild.scala`, then rebuilding Spark (`sbt/sbt clean compile`).
**Other documents:**
-* [Building Spark With Maven](building-with-maven.html): Build Spark using the Maven build tool
* [Configuration](configuration.html): customize Spark via its configuration system
* [Tuning Guide](tuning.html): best practices to optimize performance and memory use
-* [Bagel](bagel-programming-guide.html): an implementation of Google's Pregel on Spark
+* [Hardware Provisioning](hardware-provisioning.html): recommendations for cluster hardware
+* [Building Spark with Maven](building-with-maven.html): Build Spark using the Maven build tool
* [Contributing to Spark](contributing-to-spark.html)
**External resources:**
-* [Spark Homepage](http://www.spark-project.org)
-* [Mailing List](http://groups.google.com/group/spark-users): ask questions about Spark here
-* [AMP Camp](http://ampcamp.berkeley.edu/): a two-day training camp at UC Berkeley that featured talks and exercises
- about Spark, Shark, Mesos, and more. [Videos](http://ampcamp.berkeley.edu/agenda-2012),
+* [Spark Homepage](http://spark.incubator.apache.org)
+* [Mailing Lists](http://spark.incubator.apache.org/mailing-lists.html): ask questions about Spark here
+* [AMP Camps](http://ampcamp.berkeley.edu/): a series of training camps at UC Berkeley that featured talks and
+ exercises about Spark, Shark, Mesos, and more. [Videos](http://ampcamp.berkeley.edu/agenda-2012),
[slides](http://ampcamp.berkeley.edu/agenda-2012) and [exercises](http://ampcamp.berkeley.edu/exercises-2012) are
available online for free.
-* [Code Examples](http://spark-project.org/examples.html): more are also available in the [examples subfolder](https://github.com/mesos/spark/tree/master/examples/src/main/scala/spark/examples) of Spark
+* [Code Examples](http://spark.incubator.apache.org/examples.html): more are also available in the [examples subfolder](https://github.com/mesos/spark/tree/master/examples/src/main/scala/spark/examples) of Spark
* [Paper Describing Spark](http://www.cs.berkeley.edu/~matei/papers/2012/nsdi_spark.pdf)
* [Paper Describing Spark Streaming](http://www.eecs.berkeley.edu/Pubs/TechRpts/2012/EECS-2012-259.pdf)
# Community
-To get help using Spark or keep up with Spark development, sign up for the [spark-users mailing list](http://groups.google.com/group/spark-users).
+To get help using Spark or keep up with Spark development, sign up for the [user mailing list](http://spark.incubator.apache.org/mailing-lists.html).
If you're in the San Francisco Bay Area, there's a regular [Spark meetup](http://www.meetup.com/spark-users/) every few weeks. Come by to meet the developers and other users.
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index 678cd57aba..fe5334ffdc 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -3,50 +3,33 @@ layout: global
title: Launching Spark on YARN
---
-Experimental support for running over a [YARN (Hadoop
+Support for running on [YARN (Hadoop
NextGen)](http://hadoop.apache.org/docs/r2.0.2-alpha/hadoop-yarn/hadoop-yarn-site/YARN.html)
-cluster was added to Spark in version 0.6.0. This was merged into master as part of 0.7 effort.
-To build spark with YARN support, please use the hadoop2-yarn profile.
-Ex: mvn -Phadoop2-yarn clean install
+was added to Spark in version 0.6.0, and improved in 0.7.0 and 0.8.0.
-# Building spark core consolidated jar.
+# Building a YARN-Enabled Assembly JAR
-We need a consolidated spark core jar (which bundles all the required dependencies) to run Spark jobs on a yarn cluster.
-This can be built either through sbt or via maven.
+We need a consolidated Spark JAR (which bundles all the required dependencies) to run Spark jobs on a YARN cluster.
+This can be built by setting the Hadoop version and `SPARK_YARN` environment variable, as follows:
-- Building spark assembled jar via sbt.
-Enable YARN support by setting `SPARK_YARN=true` when invoking sbt:
+ SPARK_HADOOP_VERSION=2.0.5-alpha SPARK_YARN=true ./sbt/sbt assembly
- SPARK_HADOOP_VERSION=2.0.5-alpha SPARK_YARN=true ./sbt/sbt clean assembly
-
-The assembled jar would typically be something like :
-`./yarn/target/spark-yarn-assembly-0.8.0-SNAPSHOT.jar`
-
-
-- Building spark assembled jar via Maven.
- Use the hadoop2-yarn profile and execute the package target.
-
-Something like this. Ex:
-
- mvn -Phadoop2-yarn -Dhadoop.version=2.0.5-alpha clean package -DskipTests=true
-
-
-This will build the shaded (consolidated) jar. Typically something like :
-`./yarn/target/spark-yarn-bin-<VERSION>-shaded.jar`
+The assembled JAR will be something like this:
+`./assembly/target/scala-{{site.SCALA_VERSION}}/spark-assembly_{{site.SPARK_VERSION}}-hadoop2.0.5.jar`.
# Preparations
-- Building spark-yarn assembly (see above).
+- Building a YARN-enabled assembly (see above).
- Your application code must be packaged into a separate JAR file.
-If you want to test out the YARN deployment mode, you can use the current Spark examples. A `spark-examples_{{site.SCALA_VERSION}}-{{site.SPARK_VERSION}}` file can be generated by running `sbt/sbt package`. NOTE: since the documentation you're reading is for Spark version {{site.SPARK_VERSION}}, we are assuming here that you have downloaded Spark {{site.SPARK_VERSION}} or checked it out of source control. If you are using a different version of Spark, the version numbers in the jar generated by the sbt package command will obviously be different.
+If you want to test out the YARN deployment mode, you can use the current Spark examples. A `spark-examples_{{site.SCALA_VERSION}}-{{site.SPARK_VERSION}}` file can be generated by running `sbt/sbt assembly`. NOTE: since the documentation you're reading is for Spark version {{site.SPARK_VERSION}}, we are assuming here that you have downloaded Spark {{site.SPARK_VERSION}} or checked it out of source control. If you are using a different version of Spark, the version numbers in the jar generated by the sbt package command will obviously be different.
# Configuration
Most of the configs are the same for Spark on YARN as other deploys. See the Configuration page for more information on those. These are configs that are specific to SPARK on YARN.
-* `SPARK_YARN_USER_ENV`, to add environment variables to the Spark processes launched on YARN. This can be a comma separated list of environment variables. ie SPARK_YARN_USER_ENV="JAVA_HOME=/jdk64,FOO=bar"
+* `SPARK_YARN_USER_ENV`, to add environment variables to the Spark processes launched on YARN. This can be a comma separated list of environment variables, e.g. `SPARK_YARN_USER_ENV="JAVA_HOME=/jdk64,FOO=bar"`.
# Launching Spark on YARN
diff --git a/pyspark b/pyspark
index 2dba2ceb21..4941a36d0d 100755
--- a/pyspark
+++ b/pyspark
@@ -31,7 +31,7 @@ if [ ! -f "$FWDIR/RELEASE" ]; then
ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/spark-assembly*hadoop*.jar >& /dev/null
if [[ $? != 0 ]]; then
echo "Failed to find Spark assembly in $FWDIR/assembly/target" >&2
- echo "You need to compile Spark before running this program" >&2
+ echo "You need to build Spark with sbt/sbt assembly before running this program" >&2
exit 1
fi
fi
diff --git a/run-example b/run-example
index ccd4356bdf..24d83ba5cf 100755
--- a/run-example
+++ b/run-example
@@ -50,7 +50,7 @@ if [ -e "$EXAMPLES_DIR"/target/spark-examples*[0-9T].jar ]; then
fi
if [[ -z $SPARK_EXAMPLES_JAR ]]; then
echo "Failed to find Spark examples assembly in $FWDIR/examples/target" >&2
- echo "You need to compile Spark before running this program" >&2
+ echo "You need to build Spark with sbt/sbt assembly before running this program" >&2
exit 1
fi
diff --git a/spark-class b/spark-class
index 5ef3de9773..244b78b4e1 100755
--- a/spark-class
+++ b/spark-class
@@ -102,10 +102,10 @@ export JAVA_OPTS
if [ ! -f "$FWDIR/RELEASE" ]; then
# Exit if the user hasn't compiled Spark
- ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/spark-assembly*.jar >& /dev/null
+ ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/spark-assembly*hadoop*.jar >& /dev/null
if [[ $? != 0 ]]; then
echo "Failed to find Spark assembly in $FWDIR/assembly/target" >&2
- echo "You need to compile Spark before running this program" >&2
+ echo "You need to build Spark with sbt/sbt assembly before running this program" >&2
exit 1
fi
fi
--
GitLab