diff --git a/devel/hadoop2/Makefile b/devel/hadoop2/Makefile index e46445fb81b9..8e80342b76ef 100644 --- a/devel/hadoop2/Makefile +++ b/devel/hadoop2/Makefile @@ -1,108 +1,109 @@ # Created by: Dmitry Sivachenko # $FreeBSD$ # Please do not submit untested updates. Be sure to start hadoop in # distributed mode and run few map/reduce. Be sure there are no exception # in any of it's log files. This version is tested under the load and # no problems encountered so far. Thanks. PORTNAME= hadoop PORTVERSION= 2.4.1 -PORTREVISION= 4 +PORTREVISION= 5 CATEGORIES= devel java MASTER_SITES= ${MASTER_SITE_APACHE} \ LOCAL/demon/ \ http://people.freebsd.org/~demon/:maven \ http://archive.apache.org/dist/tomcat/tomcat-6/v${TOMCAT_VERSION}/bin/:tomcat \ http://dist.codehaus.org/jetty/jetty-6.1.14/:jetty MASTER_SITE_SUBDIR=${PORTNAME}/common/stable PKGNAMESUFFIX= 2 DISTFILES= ${PORTNAME}-${PORTVERSION}-src${EXTRACT_SUFX} FreeBSD-hadoop2-${PORTVERSION}-maven-repository.tar.gz:maven apache-tomcat-${TOMCAT_VERSION}.tar.gz:tomcat jetty-6.1.14.zip:jetty DIST_SUBDIR= hadoop MAINTAINER= demon@FreeBSD.org COMMENT= Apache Map/Reduce framework LICENSE= APACHE20 BUILD_DEPENDS= ${LOCALBASE}/share/java/maven3/bin/mvn:${PORTSDIR}/devel/maven3 \ cmake:${PORTSDIR}/devel/cmake \ ${LOCALBASE}/protobuf25/bin/protoc:${PORTSDIR}/devel/protobuf25 LIB_DEPENDS= libsnappy.so:${PORTSDIR}/archivers/snappy -RUN_DEPENDS= bash:${PORTSDIR}/shells/bash +RUN_DEPENDS= bash:${PORTSDIR}/shells/bash \ + ssid:${PORTSDIR}/sysutils/ssid CONFLICTS_INSTALL= apache-hadoop-1* USES= shebangfix USE_JAVA= yes JAVA_VERSION= 1.7+ USE_LDCONFIG= yes SHEBANG_FILES= hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh MAKE_ENV+= HADOOP_PROTOC_PATH=${LOCALBASE}/protobuf25/bin/protoc WRKSRC= ${WRKDIR}/${PORTNAME}-${PORTVERSION}-src TOMCAT_VERSION= 6.0.36 HADOOP_DIST= ${WRKSRC}/hadoop-dist/target/hadoop-${PORTVERSION} HADOOP_LOGDIR= /var/log/hadoop HADOOP_RUNDIR= /var/run/hadoop HDFS_USER= hdfs MAPRED_USER= mapred HADOOP_GROUP= hadoop USERS= ${HDFS_USER} ${MAPRED_USER} GROUPS= ${HADOOP_GROUP} SUB_FILES= hadoop-layout.sh USE_RC_SUBR= historyserver nodemanager resourcemanager webappproxyserver datanode namenode secondarynamenode PLIST_SUB= PORTVERSION="${PORTVERSION}" \ HADOOP_LOGDIR="${HADOOP_LOGDIR}" \ HADOOP_RUNDIR="${HADOOP_RUNDIR}" \ HDFS_USER="${HDFS_USER}" \ MAPRED_USER="${MAPRED_USER}" \ HADOOP_GROUP="${HADOOP_GROUP}" SUB_LIST= HDFS_USER="${HDFS_USER}" \ MAPRED_USER="${MAPRED_USER}" \ HADOOP_GROUP="${HADOOP_GROUP}" \ JAVA_HOME="${JAVA_HOME}" \ HADOOP_LOGDIR="${HADOOP_LOGDIR}" \ HADOOP_RUNDIR="${HADOOP_RUNDIR}" post-patch: ${REINPLACE_CMD} -e "s#/bin/bash#${LOCALBASE}/bin/bash#" ${WRKSRC}/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java ${WRKSRC}/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java ${WRKSRC}/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java ${WRKSRC}/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java do-build: ${MKDIR} ${WRKSRC}/hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads ${CP} ${DISTDIR}/${DIST_SUBDIR}/apache-tomcat-${TOMCAT_VERSION}.tar.gz ${WRKSRC}/hadoop-hdfs-project/hadoop-hdfs-httpfs/downloads/ cd ${WRKSRC} && ${SETENV} ${MAKE_ENV} ${LOCALBASE}/share/java/maven3/bin/mvn -Dmaven.repo.local=${WRKDIR}/m2 --offline clean package -Pdist,native -DskipTests -Drequire.snappy -Dsnappy.prefix=${LOCALBASE} post-build: ${RM} ${HADOOP_DIST}/etc/hadoop/*.cmd ${MV} ${HADOOP_DIST}/bin/rcc ${HADOOP_DIST}/bin/hadoop-rcc # With jetty-6.1.26 tasktracker's threads hung with the following error: # org.mortbay.io.nio.SelectorManager$SelectSet@abdcc1c JVM BUG(s) - injecting delay 59 times # See https://issues.apache.org/jira/browse/MAPREDUCE-2386 .for dir in share/hadoop/common/lib share/hadoop/hdfs/lib share/hadoop/yarn/lib share/hadoop/tools/lib ${RM} ${HADOOP_DIST}/${dir}/jetty-util-6.1.26.jar ${HADOOP_DIST}/${dir}/jetty-6.1.26.jar ${CP} ${WRKDIR}/jetty-6.1.14/lib/jetty-6.1.14.jar ${WRKDIR}/jetty-6.1.14/lib/jetty-util-6.1.14.jar ${HADOOP_DIST}/${dir}/ .endfor do-install: cd ${HADOOP_DIST}/bin && ${INSTALL_SCRIPT} hadoop hdfs mapred hadoop-rcc yarn ${STAGEDIR}${PREFIX}/bin/ cd ${HADOOP_DIST} && ${COPYTREE_BIN} "libexec sbin" ${STAGEDIR}${PREFIX}/ "! -name *.cmd" cd ${HADOOP_DIST}/include && ${INSTALL_DATA} *h ${STAGEDIR}${PREFIX}/include/ cd ${HADOOP_DIST}/lib/native && ${INSTALL_DATA} *.a ${STAGEDIR}${PREFIX}/lib/ cd ${HADOOP_DIST}/lib/native && ${INSTALL_DATA} libhadoop.so.1.0.0 ${STAGEDIR}${PREFIX}/lib/libhadoop.so.1 cd ${HADOOP_DIST}/lib/native && ${INSTALL_DATA} libhdfs.so.0.0.0 ${STAGEDIR}${PREFIX}/lib/libhdfs.so.0 ${LN} -sf libhdfs.so.0 ${STAGEDIR}${PREFIX}/lib/libhdfs.so ${LN} -sf libhadoop.so.1 ${STAGEDIR}${PREFIX}/lib/libhadoop.so cd ${HADOOP_DIST}/share/hadoop && ${COPYTREE_SHARE} "*" ${STAGEDIR}${DATADIR}/ "! -name *-sources.jar -and ! -name sources" ${MKDIR} ${STAGEDIR}${EXAMPLESDIR}/conf cd ${HADOOP_DIST}/etc/hadoop && ${COPYTREE_SHARE} "*" ${STAGEDIR}${EXAMPLESDIR}/conf/ ${INSTALL_DATA} ${WRKSRC}/hadoop-hdfs-project/hadoop-hdfs/target/classes/hdfs-default.xml ${WRKSRC}/hadoop-hdfs-project/hadoop-hdfs-httpfs/target/classes/httpfs-default.xml ${WRKSRC}/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/target/classes/yarn-default.xml ${WRKSRC}/hadoop-common-project/hadoop-common/target/classes/core-default.xml ${WRKSRC}/hadoop-tools/hadoop-distcp/target/classes/distcp-default.xml ${WRKSRC}/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/target/classes/mapred-default.xml ${STAGEDIR}/${EXAMPLESDIR}/ ${INSTALL_DATA} ${WRKDIR}/hadoop-layout.sh ${STAGEDIR}${PREFIX}/libexec/ ${MKDIR} ${STAGEDIR}${ETCDIR} ${MKDIR} ${STAGEDIR}${HADOOP_LOGDIR} ${MKDIR} ${STAGEDIR}${HADOOP_RUNDIR} .include diff --git a/devel/hadoop2/files/patch-hadoop-common-project-hadoop-common-src-main-java-org-apache-hadoop-util-Shell.java b/devel/hadoop2/files/patch-hadoop-common-project-hadoop-common-src-main-java-org-apache-hadoop-util-Shell.java new file mode 100644 index 000000000000..89a0fa4ded2f --- /dev/null +++ b/devel/hadoop2/files/patch-hadoop-common-project-hadoop-common-src-main-java-org-apache-hadoop-util-Shell.java @@ -0,0 +1,21 @@ +--- ./hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java.orig 2014-06-21 09:40:07.000000000 +0400 ++++ ./hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java 2015-01-06 01:21:22.000000000 +0300 +@@ -348,15 +348,15 @@ abstract public class Shell { + ShellCommandExecutor shexec = null; + boolean setsidSupported = true; + try { +- String[] args = {"setsid", "bash", "-c", "echo $$"}; ++ String[] args = {"ssid", "bash", "-c", "echo $$"}; + shexec = new ShellCommandExecutor(args); + shexec.execute(); + } catch (IOException ioe) { +- LOG.debug("setsid is not available on this machine. So not using it."); ++ LOG.debug("ssid is not available on this machine. So not using it."); + setsidSupported = false; + } finally { // handle the exit code + if (LOG.isDebugEnabled()) { +- LOG.debug("setsid exited with exit code " ++ LOG.debug("ssid exited with exit code " + + (shexec != null ? shexec.getExitCode() : "(null executor)")); + } + } diff --git a/devel/hadoop2/files/patch-hadoop-mapreduce-client-core-src-main-java-org-apache-hadoop-mapred-TaskLog.java b/devel/hadoop2/files/patch-hadoop-mapreduce-client-core-src-main-java-org-apache-hadoop-mapred-TaskLog.java new file mode 100644 index 000000000000..86928a25ba47 --- /dev/null +++ b/devel/hadoop2/files/patch-hadoop-mapreduce-client-core-src-main-java-org-apache-hadoop-mapred-TaskLog.java @@ -0,0 +1,11 @@ +--- ./hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java.orig 2015-01-06 01:14:31.000000000 +0300 ++++ ./hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLog.java 2015-01-06 01:14:47.000000000 +0300 +@@ -538,7 +538,7 @@ public class TaskLog { + mergedCmd.append("("); + } else if(ProcessTree.isSetsidAvailable && useSetsid && + !Shell.WINDOWS) { +- mergedCmd.append("exec setsid "); ++ mergedCmd.append("exec ssid "); + } else { + mergedCmd.append("exec "); + } diff --git a/devel/hadoop2/files/patch-hadoop-mapreduce-client-core-src-main-java-org-apache-hadoop-mapreduce-util-ProcessTree.java b/devel/hadoop2/files/patch-hadoop-mapreduce-client-core-src-main-java-org-apache-hadoop-mapreduce-util-ProcessTree.java new file mode 100644 index 000000000000..5cea6111db95 --- /dev/null +++ b/devel/hadoop2/files/patch-hadoop-mapreduce-client-core-src-main-java-org-apache-hadoop-mapreduce-util-ProcessTree.java @@ -0,0 +1,20 @@ +--- ./hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java.orig 2015-01-06 01:16:49.000000000 +0300 ++++ ./hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ProcessTree.java 2015-01-06 01:17:24.000000000 +0300 +@@ -54,14 +54,14 @@ public class ProcessTree { + ShellCommandExecutor shexec = null; + boolean setsidSupported = true; + try { +- String[] args = {"setsid", "bash", "-c", "echo $$"}; ++ String[] args = {"ssid", "bash", "-c", "echo $$"}; + shexec = new ShellCommandExecutor(args); + shexec.execute(); + } catch (IOException ioe) { +- LOG.warn("setsid is not available on this machine. So not using it."); ++ LOG.warn("ssid is not available on this machine. So not using it."); + setsidSupported = false; + } finally { // handle the exit code +- LOG.info("setsid exited with exit code " + shexec.getExitCode()); ++ LOG.info("ssid exited with exit code " + shexec.getExitCode()); + } + return setsidSupported; + } diff --git a/devel/hadoop2/files/patch-src-main-java-org-apache-hadoop-yarn-server-nodemanager-DefaultContainerExecutor.java b/devel/hadoop2/files/patch-src-main-java-org-apache-hadoop-yarn-server-nodemanager-DefaultContainerExecutor.java new file mode 100644 index 000000000000..8731545b8b8f --- /dev/null +++ b/devel/hadoop2/files/patch-src-main-java-org-apache-hadoop-yarn-server-nodemanager-DefaultContainerExecutor.java @@ -0,0 +1,11 @@ +--- ./hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java.orig 2015-01-06 01:28:53.000000000 +0300 ++++ ./hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java 2015-01-06 01:29:32.000000000 +0300 +@@ -277,7 +277,7 @@ public class DefaultContainerExecutor ex + pout.println(); + pout.println("echo $$ > " + pidFile.toString() + ".tmp"); + pout.println("/bin/mv -f " + pidFile.toString() + ".tmp " + pidFile); +- String exec = Shell.isSetsidAvailable? "exec setsid" : "exec"; ++ String exec = Shell.isSetsidAvailable? "exec ssid" : "exec"; + pout.println(exec + " /bin/bash \"" + + launchDst.toUri().getPath().toString() + "\""); + }