103 lines
1.8 KiB
Markdown
103 lines
1.8 KiB
Markdown
|
---
|
|||
|
|
title: hadoop的使用
|
|||
|
|
tags: default
|
|||
|
|
categories: default
|
|||
|
|
date: 2022-04-08 10:39:43
|
|||
|
|
|
|||
|
|
---
|
|||
|
|
|
|||
|
|
# 什么是hadoop
|
|||
|
|
|
|||
|
|
> 分布式大数据平台
|
|||
|
|
|
|||
|
|
# hadoop组件
|
|||
|
|
|
|||
|
|
+ hdfs -- `分布式文件系统`
|
|||
|
|
+ have -- `数据仓库`
|
|||
|
|
+ HBase -- `分布式数据库`
|
|||
|
|
|
|||
|
|
# hadoop的搭建
|
|||
|
|
|
|||
|
|
> master ---主要节点 -- 内存4500MB --cpu一个 -- 硬盘40Gb -- ip地址 10.0.0.100
|
|||
|
|
|
|||
|
|
> slave --- 从节点 --内存 2500MB --cpu一个 --硬盘40Gb -- ip地址 10.0.0.200
|
|||
|
|
|
|||
|
|
```
|
|||
|
|
systemtcl stop firewalled && systemtcl disable firewalled关闭防火墙
|
|||
|
|
/etc/selinux/config 关闭linux
|
|||
|
|
setenforce 0 刷新配置文件
|
|||
|
|
/etc/hosts 本地host文件
|
|||
|
|
/etc/ntp.conf 时间配置文件
|
|||
|
|
scp source目录 用户名@ip：源文件目录 # 远程copy文件
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
```
|
|||
|
|
ntpdate # 修改时间的工具
|
|||
|
|
ntp1.aliyun.com 阿里时间服务器
|
|||
|
|
ntpdate ntp1.aliyun.com 配置时间服务器
|
|||
|
|
clock -w 保存当前时间
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
## 修改主机名
|
|||
|
|
|
|||
|
|
```
|
|||
|
|
hostnamectl set-hostname 主机名
|
|||
|
|
bash #刷新
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
## 任务计划
|
|||
|
|
|
|||
|
|
```
|
|||
|
|
crontab -e 书写任务计划
|
|||
|
|
crontab -l 查询任务计划
|
|||
|
|
*/1 * * * * /sbin/ntp
|
|||
|
|
systemctl restart crond 重启计划任务
|
|||
|
|
PATH=$JAVA_HOME/bin:$PATh
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
## 设置本地yum源
|
|||
|
|
|
|||
|
|
*/etc/yum.repo.d*
|
|||
|
|
|
|||
|
|
```
|
|||
|
|
[hadoop]
|
|||
|
|
name=hadoop
|
|||
|
|
baseurl=file:///
|
|||
|
|
enable=1
|
|||
|
|
gpgcheck=1
|
|||
|
|
|
|||
|
|
|
|||
|
|
yum respse-list 查看yun源
|
|||
|
|
```
|
|||
|
|
|
|||
|
|
```
|
|||
|
|
09:52
|
|||
|
|
Absinthe
|
|||
|
|
任务计划 crontab -e 书写任务计划
|
|||
|
|
crontab -l 查询任务计划
|
|||
|
|
|
|||
|
|
ntpdate
|
|||
|
|
|
|||
|
|
*/1 * * * * /sbin/ntpdate s1 >> /var/log/ntpdate.log
|
|||
|
|
|
|||
|
|
|
|||
|
|
hadoop java ---jdk
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
|
|||
|
|
mariadb mariadb-server mysql-connector-java
|
|||
|
|
ambari-server
|
|||
|
|
|
|||
|
|
|
|||
|
|
create database ambari; ---创建一个库
|
|||
|
|
|
|||
|
|
grant all privileges on ambari.* to 'ambari'@'localhost' identified by 'bigdata';
|
|||
|
|
|
|||
|
|
---为数据库创建一个用户 ambari 密码为 bigdata
|
|||
|
|
grant all privileges on ambari.* to 'ambari'@'%' identified by 'bigdata';
|
|||
|
|
---把权限交给ambari
|
|||
|
|
```
|