forked from Stratio/deep-spark
-
Notifications
You must be signed in to change notification settings - Fork 0
/
make-distribution-deep.sh
executable file
·129 lines (84 loc) · 3.7 KB
/
make-distribution-deep.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
#!/bin/bash
# Stratio Deep Deployment script
TMPDIR=/tmp/stratio-deep-distribution
rm -rf ${TMPDIR}
mkdir -p ${TMPDIR}
export JAVA_HOME=$JAVA_HOME
export PATH=$JAVA_HOME/bin:$PATH
export SCALA_HOME=$SCALA_HOME
export PATH=$SCALA_HOME/bin:$PATH
SPARK_REPO="$1"
if [ -z "$1" ]; then
SPARK_REPO="https://github.com/Stratio/spark.git"
fi
SPARK_BRANCH="$2"
LOCAL_EDITOR=$(which vim)
if [ -z "$LOCAL_EDITOR" ]; then
$LOCAL_EDITOR=$(which vi)
fi
if [ -z "$LOCAL_EDITOR" ]; then
echo "Cannot find any command line editor, ChangeLog.txt won't be edited interactively"
fi
if [ -z "$2" ]; then
SPARK_BRANCH="stratio-branch-1.3.1"
fi
echo "SPARK_REPO: ${SPARK_REPO}"
echo "SPARK_BRANCH: ${SPARK_BRANCH}"
echo " >>> STRATIO DEEP MAKE DISTRIBUTION <<< "
LOCAL_DIR=`pwd`
echo "LOCAL_DIR=$LOCAL_DIR"
mvn -version >/dev/null || { echo "Cannot find Maven in path, aborting"; exit 1; }
cd ../deep-parent
RELEASE_VER=$(mvn org.apache.maven.plugins:maven-help-plugin:2.1.1:evaluate -Dexpression=project.version 2>/dev/null | grep -v '\[') || { echo "Cannot obtain project version, aborting"; exit 1; }
echo "RELEASE_VER: ${RELEASE_VER}"
if [ "$RELEASE_VER" = "" ]; then
echo "Release version empty, aborting"; exit 1;
fi
#### Create Deep jars from github (master tag) through maven release plugin
echo "################################################"
echo "Compiling Deep"
echo "################################################"
echo "$(pwd)"
mvn clean install package -DskipTests || { echo "Cannot build Deep project, aborting"; exit 1; }
mkdir -p ${TMPDIR}/lib || { echo "Cannot create output lib directory"; exit 1; }
cp -u ../*/target/*.jar ${TMPDIR}/lib || { echo "Cannot copy target jars to output lib directory, aborting"; exit 1; }
cp -u ../*/target/alternateLocation/*.jar ${TMPDIR}/lib || { echo "Cannot copy alternate jars to output lib directory, aborting"; exit 1; }
git fetch --tags
latest_tag=$(git describe --tags `git rev-list --tags --max-count=1`)
echo -e "[${RELEASE_VER}]\n\n$(git log ${latest_tag}..HEAD)\n\n$(cat ChangeLog.txt)" > ${TMPDIR}/ChangeLog.txt
#if [ -n "$LOCAL_EDITOR" ]; then
# $LOCAL_EDITOR ${TMPDIR}/ChangeLog.txt
#fi
#### Create Deep external extractors jars from github
echo "################################################"
echo "Compiling Deep external extractors"
echo "################################################"
../deep-scripts/make-distribution-deep-external.sh ${TMPDIR}
echo "################################################"
echo "Creating Spark distribuition"
echo "################################################"
cd ${TMPDIR}
STRATIOSPARKDIR=stratiospark
git clone "$SPARK_REPO" ${STRATIOSPARKDIR} || { echo "Cannot clone Spark project from repository: ${SPARK_REPO}"; exit 1; }
cd ./${STRATIOSPARKDIR}/
git checkout "$SPARK_BRANCH" || { echo "Cannot checkout branch: ${SPARK_BRANCH}"; exit 1; }
chmod +x bin/stratio-deep-shell
#--hadoop 2.0.0-mr1-cdh4.4.0
./make-distribution.sh --skip-java-test -Dhadoop.version=2.4.0 -Pyarn -Phive -Pnetlib-lgpl -Pscala-2.10 || { echo "Cannot make Spark distribution"; exit 1; }
cd ..
DISTDIR=spark-deep-distribution-${RELEASE_VER}
DISTFILENAME=${DISTDIR}.tgz
cp ${TMPDIR}/lib/*.jar ${STRATIOSPARKDIR}/dist/lib/
rm -f ${STRATIOSPARKDIR}/dist/lib/*-sources.jar
rm -f ${STRATIOSPARKDIR}/dist/lib/*-javadoc.jar
rm -f ${STRATIOSPARKDIR}/dist/lib/*-tests.jar
mv ${STRATIOSPARKDIR}/dist/ ${DISTDIR}
cp ${TMPDIR}/ChangeLog.txt ${DISTDIR}/
echo "DISTFILENAME: ${DISTFILENAME}"
tar czf ${DISTFILENAME} ${DISTDIR} || { echo "Cannot create tgz"; exit 1; }
mv ${DISTFILENAME} ${LOCAL_DIR}
echo "################################################"
echo "Finishing process"
echo "################################################"
cd ${LOCAL_DIR}
rm -rf ${TMPDIR}