Hi @Scharan
Spark conf:
tee -a ~/spark/conf/spark-defaults.conf >> /dev/null <<EOF
spark.sql.catalogImplementation hive
spark.master yarn
spark.driver.memory 4g
spark.shuffle.service.enabled true
spark.yarn.jars hdfs:///user/zeppelin/lib/spark/jars/*
EOF
Livy conf:
tee -a ~/livy/conf/livy-env.sh >> /dev/null <<EOF
JAVA_HOME=/usr/lib/jvm/java-8-oracle
HADOOP_HOME=/usr/lib/hadoop
HADOOP_CONF_DIR=/etc/hadoop/conf
SPARK_HOME=~/spark
LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native
EOF
tee -a ~/livy/conf/livy.conf >> /dev/null <<EOF
livy.repl.enable-hive-context = true
livy.spark.master = yarn
livy.spark.deploy-mode = cluster
livy.impersonation.enabled = true
EOF
Zeppelin Conf:
tee ~/zeppelin/conf/shiro.ini > /dev/null <<EOF
[main]
### FreeIPA over LDAP
ldapRealm = org.apache.zeppelin.realm.LdapRealm
ldapRealm.contextFactory.environment[ldap.searchBase] = dc=my,dc=corp,dc=de
ldapRealm.userDnTemplate = uid={0},cn=users,cn=accounts,dc=my,dc=corp,dc=de
ldapRealm.userSearchScope = subtree
ldapRealm.groupSearchScope = subtree
ldapRealm.searchBase = cn=accounts,dc=my,dc=corp,dc=de
ldapRealm.userSearchBase = cn=users,cn=accounts,dc=my,dc=corp,dc=de
ldapRealm.groupSearchBase = cn=groups,cn=accounts,dc=my,dc=corp,dc=de
ldapRealm.userObjectClass = person
ldapRealm.groupObjectClass = groupofnames
ldapRealm.groupSearchEnableMatchingRuleInChain = true
ldapRealm.userSearchAttributeName = uid
ldapRealm.userSearchFilter=(&(objectclass=person)(uid={0}))
ldapRealm.memberAttribute = member
ldapRealm.memberAttributeValueTemplate = uid={0},cn=users,cn=accounts,dc=my,dc=corp,dc=de
ldapRealm.contextFactory.authenticationMechanism = simple
ldapRealm.contextFactory.systemUsername = zeppelin
ldapRealm.contextFactory.systemPassword = password
ldapRealm.contextFactory.url = ldap://freeipa.my.corp.de:389
securityManager.realms = \$ldapRealm
sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager
### Enables 'HttpOnly' flag in Zeppelin cookies
cookie = org.apache.shiro.web.servlet.SimpleCookie
cookie.name = JSESSIONID
cookie.httpOnly = true
### Uncomment the below line only when Zeppelin is running over HTTPS
#cookie.secure = true
sessionManager.sessionIdCookie = \$cookie
securityManager.sessionManager = \$sessionManager
# 86,400,000 milliseconds = 24 hour
securityManager.sessionManager.globalSessionTimeout = 86400000
shiro.loginUrl = /api/login
[roles]
pharos = *
admin = *
[urls]
/api/version = anon
# Allow all authenticated users to restart interpreters on a notebook page.
# Comment out the following line if you would like to authorize only admin users to restart interpreters.
/api/interpreter/setting/restart/** = authc
/api/interpreter/** = authc
/api/configurations/** = authc
/api/credential/** = authc
/** = authc
EOF
Interpreter:
"livy": {
"id": "livy",
"name": "livy",
"group": "livy",
"properties": {
"livy.spark.executor.instances": {
"name": "livy.spark.executor.instances",
"value": "",
"type": "number"
},
"livy.spark.dynamicAllocation.cachedExecutorIdleTimeout": {
"name": "livy.spark.dynamicAllocation.cachedExecutorIdleTimeout",
"value": "",
"type": "string"
},
"zeppelin.livy.concurrentSQL": {
"name": "zeppelin.livy.concurrentSQL",
"value": false,
"type": "checkbox"
},
"zeppelin.livy.url": {
"name": "zeppelin.livy.url",
"value": "http://localhost:8998",
"type": "url"
},
"zeppelin.livy.pull_status.interval.millis": {
"name": "zeppelin.livy.pull_status.interval.millis",
"value": "1000",
"type": "number"
},
"livy.spark.executor.memory": {
"name": "livy.spark.executor.memory",
"value": "",
"type": "string"
},
"zeppelin.livy.restart_dead_session": {
"name": "zeppelin.livy.restart_dead_session",
"value": false,
"type": "checkbox"
},
"livy.spark.dynamicAllocation.enabled": {
"name": "livy.spark.dynamicAllocation.enabled",
"value": false,
"type": "checkbox"
},
"zeppelin.livy.maxLogLines": {
"name": "zeppelin.livy.maxLogLines",
"value": "1000",
"type": "number"
},
"livy.spark.dynamicAllocation.minExecutors": {
"name": "livy.spark.dynamicAllocation.minExecutors",
"value": "",
"type": "number"
},
"livy.spark.executor.cores": {
"name": "livy.spark.executor.cores",
"value": "",
"type": "number"
},
"zeppelin.livy.session.create_timeout": {
"name": "zeppelin.livy.session.create_timeout",
"value": "120",
"type": "number"
},
"zeppelin.livy.spark.sql.maxResult": {
"name": "zeppelin.livy.spark.sql.maxResult",
"value": "1000",
"type": "number"
},
"livy.spark.driver.cores": {
"name": "livy.spark.driver.cores",
"value": "4",
"type": "number"
},
"livy.spark.jars.packages": {
"name": "livy.spark.jars.packages",
"value": "",
"type": "textarea"
},
"zeppelin.livy.spark.sql.field.truncate": {
"name": "zeppelin.livy.spark.sql.field.truncate",
"value": true,
"type": "checkbox"
},
"livy.spark.driver.memory": {
"name": "livy.spark.driver.memory",
"value": "8G",
"type": "string"
},
"zeppelin.livy.displayAppInfo": {
"name": "zeppelin.livy.displayAppInfo",
"value": true,
"type": "checkbox"
},
"zeppelin.livy.principal": {
"name": "zeppelin.livy.principal",
"value": "",
"type": "string"
},
"zeppelin.livy.keytab": {
"name": "zeppelin.livy.keytab",
"value": "",
"type": "textarea"
},
"livy.spark.dynamicAllocation.maxExecutors": {
"name": "livy.spark.dynamicAllocation.maxExecutors",
"value": "",
"type": "number"
},
"livy.spark.dynamicAllocation.initialExecutors": {
"name": "livy.spark.dynamicAllocation.initialExecutors",
"value": "",
"type": "number"
}
},
"status": "READY",
"interpreterGroup": [
{
"name": "spark",
"class": "org.apache.zeppelin.livy.LivySparkInterpreter",
"defaultInterpreter": true,
"editor": {
"language": "scala",
"editOnDblClick": false,
"completionKey": "TAB",
"completionSupport": true
}
},
{
"name": "sql",
"class": "org.apache.zeppelin.livy.LivySparkSQLInterpreter",
"defaultInterpreter": false,
"editor": {
"language": "sql",
"editOnDblClick": false,
"completionKey": "TAB",
"completionSupport": true
}
},
{
"name": "pyspark",
"class": "org.apache.zeppelin.livy.LivyPySparkInterpreter",
"defaultInterpreter": false,
"editor": {
"language": "python",
"editOnDblClick": false,
"completionKey": "TAB",
"completionSupport": true
}
},
{
"name": "pyspark3",
"class": "org.apache.zeppelin.livy.LivyPySpark3Interpreter",
"defaultInterpreter": false,
"editor": {
"language": "python",
"editOnDblClick": false,
"completionKey": "TAB",
"completionSupport": true
}
},
{
"name": "sparkr",
"class": "org.apache.zeppelin.livy.LivySparkRInterpreter",
"defaultInterpreter": false,
"editor": {
"language": "r",
"editOnDblClick": false,
"completionKey": "TAB",
"completionSupport": true
}
},
{
"name": "shared",
"class": "org.apache.zeppelin.livy.LivySharedInterpreter",
"defaultInterpreter": false
}
],
"dependencies": [],
"option": {
"remote": true,
"port": -1,
"perNote": "shared",
"perUser": "scoped",
"isExistingProcess": false,
"setPermission": false,
"owners": [],
"isUserImpersonate": false
}
},
Thanks!
Nicola
