文章/答案/技术大牛

发布

社区首页 >问答首页 >Hue访问HDFS:绕过默认的hue.ini？

问Hue访问HDFS:绕过默认的hue.ini？
EN

Stack Overflow用户

提问于 2019-07-19 16:28:33

回答 2查看 1.1K关注 0票数 0

布设

我试图用bde2020 (学习目的)提供的图像组成一个轻量级的最小hadoop堆栈。现在，堆栈包括(除其他外)

namenode
达塔诺特
色调

基本上，我从大数据欧洲正式船坞组成开始，并添加了一个基于他们的文件的色调图像

问题

Hue的文件浏览器无法访问HDFS：

Cannot access: /user/dav. The HDFS REST service is not available. Note: you are a Hue admin but not a HDFS superuser, "hdfs" or part of HDFS supergroup, "supergroup".

HTTPConnectionPool(host='namenode', port=50070): Max retries exceeded with url: /webhdfs/v1/user/dav?op=GETFILESTATUS&user.name=hue&doas=dav (Caused by NewConnectionError('<requests.packages.urllib3.connection.HTTPConnection object at 0x7f8119a3cf10>: Failed to establish a new connection: [Errno 111] Connection refused',))

到目前为止，我试图对这个问题进行界定

显式地将同一网络上的所有服务
将dfs_webhdfs_url指向namenode文件(来源)中的localhost:9870/webhdfs/v1，并相应地在色调容器中编辑hue.ini (添加webhdfs_url=http://namenode:9870/webhdfs/v1)

当我登录到hue的容器时，我可以看到namenode的端口9870是打开的(nmap -p 9870 namenode)。50070不是。我不认为我的问题与网络有关。尽管编辑了hue.ini，Hue仍然选择端口50070。那么，在我当前的设置中，如何强制hue去端口9870呢？(如果这就是原因)

码头工

version: '3.7'

services:
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.1-java8
    container_name: namenode
    hostname: namenode
    domainname: hadoop
    ports:
      - 9870:9870
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
      - ./entrypoints/namenode/entrypoint.sh:/entrypoint.sh
    env_file:
      - ./hadoop.env
      - .env
    networks:
      - hadoop_net
    # TODO adduser --ingroup hadoop dav

  datanode1:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.1-java8
    container_name: datanode
    hostname: datanode1
    domainname: hadoop
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    env_file:
      - ./hadoop.env
    networks:
      - hadoop_net

  resourcemanager:
    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.1-java8
    container_name: resourcemanager
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864"
    env_file:
      - ./hadoop.env
    networks:
      - hadoop_net

  nodemanager1:
    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.1-java8
    container_name: nodemanager
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
    env_file:
      - ./hadoop.env
    networks:
      - hadoop_net    

  historyserver:
    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.1-java8
    container_name: historyserver
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088"
    volumes:
      - hadoop_historyserver:/hadoop/yarn/timeline
    env_file:
      - ./hadoop.env
    networks:
      - hadoop_net

  filebrowser: 
    container_name: hue
    image: bde2020/hdfs-filebrowser:3.11
    ports:
      - "8088:8088"
    env_file:
      - ./hadoop.env
    volumes: # BYPASS DEFAULT webhdfs url
      - ./overrides/hue/hue.ini:/opt/hue/desktop/conf.dist/hue.ini
    environment:
      - NAMENODE_HOST=namenode
    networks:
      - hadoop_net

networks:
  hadoop_net: 

volumes:
  hadoop_namenode:
  hadoop_datanode:
  hadoop_historyserver:

docker

hadoop

docker-compose

hdfs

hue

回答 2

Stack Overflow用户

回答已采纳

发布于 2019-07-20 11:06:24

是啊找到了。几个关键要素：

在hadoop 3.*中，webhdfs不再听50070，但9870是标准端口。
重写hue.ini需要挂载一个名为hue-overrides.ini的文件
来自gethue的色调图像比来自bde2020 (他们的hadoop堆栈岩石)的色调图像更有时代感。

码头工

version: '3.7'

services:
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.1.1-java8
    container_name: namenode
    ports:
      - 9870:9870
      - 8020:8020
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
      - ./overrides/namenode/entrypoint.sh:/entrypoint.sh
    env_file:
      - ./hadoop.env
      - .env
    networks:
      - hadoop

  filebrowser: 
    container_name: hue
    image: gethue/hue:4.4.0
    ports:
      - "8000:8888"
    env_file:
      - ./hadoop.env
    volumes: # HERE
      - ./overrides/hue/hue-overrides.ini:/usr/share/hue/desktop/conf/hue-overrides.ini
    depends_on:
      - namenode
    networks:
      - hadoop
      - frontend

  datanode1:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.1.1-java8
    container_name: datanode1
    volumes:
      - hadoop_datanode:/hadoop/dfs/data
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    env_file:
      - ./hadoop.env
    networks:
      - hadoop

  resourcemanager:
    image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.1.1-java8
    container_name: resourcemanager
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode1:9864"
    env_file:
      - ./hadoop.env
    networks:
      - hadoop

  nodemanager1:
    image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.1.1-java8
    container_name: nodemanager
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode1:9864 resourcemanager:8088"
    env_file:
      - ./hadoop.env
    networks:
      - hadoop    

  historyserver:
    image: bde2020/hadoop-historyserver:2.0.0-hadoop3.1.1-java8
    container_name: historyserver
    environment:
      SERVICE_PRECONDITION: "namenode:9870 datanode1:9864 resourcemanager:8088"
    volumes:
      - hadoop_historyserver:/hadoop/yarn/timeline
    env_file:
      - ./hadoop.env
    networks:
      - hadoop

networks:
  hadoop: 
  frontend:

volumes:
  hadoop_namenode:
  hadoop_datanode:
  hadoop_historyserver:

hadoop.env

CORE_CONF_fs_defaultFS=hdfs://namenode:8020
CORE_CONF_hadoop_http_staticuser_user=root
CORE_CONF_hadoop_proxyuser_hue_hosts=*
CORE_CONF_hadoop_proxyuser_hue_groups=*
CORE_CONF_io_compression_codecs=org.apache.hadoop.io.compress.SnappyCodec

HDFS_CONF_dfs_replication=1
HDFS_CONF_dfs_webhdfs_enabled=true
HDFS_CONF_dfs_permissions_enabled=false
HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false

hue-overrides.ini

[desktop]
  http_host=0.0.0.0
  http_port=8888
  time_zone=France
  dev=true
  app_blacklist=impala,zookeeper,oozie,hbase,security,search
[hadoop]
  [[hdfs_clusters]]
    [[[default]]]
      fs_defaultfs=hdfs://namenode:8020
      webhdfs_url=http://namenode:9870/webhdfs/v1
      security_enabled=false

谢谢@cricket_007

票数 0

Stack Overflow用户

发布于 2019-07-19 21:09:41

我能够让Filebrowser使用这个INI

[desktop]
  http_host=0.0.0.0
  http_port=8888
  time_zone=America/Chicago
  dev=true
  app_blacklist=impala,zookeeper,oozie,hbase,security,search
[hadoop]
  [[hdfs_clusters]]
    [[[default]]]
      fs_defaultfs=hdfs://namenode:8020
      webhdfs_url=http://namenode:50070/webhdfs/v1
      security_enabled=false

而这篇作文

version: "2"

services:
  namenode:
    image: bde2020/hadoop-namenode:1.1.0-hadoop2.7.1-java8
    container_name: namenode
    ports:
      - 8020:8020
      - 50070:50070
      # - 59050:59050
    volumes:
      - hadoop_namenode:/hadoop/dfs/name
    environment:
      - CLUSTER_NAME=test
    env_file:
      - ./hadoop.env
    networks:
      - hadoop

  datanode1:
    image: bde2020/hadoop-datanode:1.1.0-hadoop2.7.1-java8
    container_name: datanode1
    ports:
      - 50075:50075
      # - 50010:50010
      # - 50020:50020
    depends_on:
      - namenode
    volumes:
      - hadoop_datanode1:/hadoop/dfs/data
    env_file:
      - ./hadoop.env
    networks:
      - hadoop

  hue:
    image: gethue/hue
    container_name: hue
    ports:
      - 8000:8888
    depends_on:
      - namenode
    volumes:
      - ./conf/hue.ini:/hue/desktop/conf/pseudo-distributed.ini
    networks:
      - hadoop
      - frontend

    volumes:
      hadoop_namenode:
      hadoop_datanode1:

    networks:
      hadoop:
      frontend:

hadoop.env还必须添加色调作为代理用户。

CORE_CONF_fs_defaultFS=hdfs://namenode:8020
CORE_CONF_hadoop_http_staticuser_user=root

CORE_CONF_hadoop_proxyuser_hue_hosts=*
CORE_CONF_hadoop_proxyuser_hue_groups=*

HDFS_CONF_dfs_replication=1
HDFS_CONF_dfs_webhdfs_enabled=true
HDFS_CONF_dfs_permissions_enabled=false

票数 4

页面原文内容由Stack Overflow提供。腾讯云小微IT领域专用引擎提供翻译支持

原文链接：

https://stackoverflow.com/questions/57116402

复制

相似问题

问Hue访问HDFS:绕过默认的hue.ini？
EN

回答 2

Stack Overflow用户

Stack Overflow用户

社区

活动

圈层

关于

腾讯云开发者

热门产品

热门推荐

更多推荐

问Hue访问HDFS:绕过默认的hue.ini？EN

回答 2

Stack Overflow用户

Stack Overflow用户

社区

活动

圈层

关于

腾讯云开发者

热门产品

热门推荐

更多推荐

问Hue访问HDFS:绕过默认的hue.ini？
EN