Oracle 11gr2 rac 修复单节点故障（删除和添加节点）

来源：互联网发布：steam饥荒mac怎么汉化编辑：程序博客网时间：2024/05/13 09:53

由于本机硬盘故障导致单个节点的/u01目录丢失。导致单节点的所有oracle信息丢失。本文章同样适用于删除和添加节点

1、RAC信息

1.1、host信息

[grid@rac1 ~]$ cat /etc/hosts
# Do not remove the following line, or various programs
# that require network functionality will fail.
127.0.0.1 localhost.localdomain localhost
::1 localhost6.localdomain6 localhost6
#Public ip
10.10.10.121 rac1
10.10.10.122 rac2
#Private ip
192.168.1.121 rac1-priv
192.168.1.122 rac2-priv
#Virtual ip
10.10.10.123 rac1-vip
10.10.10.124 rac2-vip
#Scan ip
10.10.10.125 rac-scan

1.2、RAC状态

[grid@rac1 ~]$ crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.DATA.dg ora....up.type ONLINE ONLINE rac1
ora....ER.lsnr ora....er.type ONLINE ONLINE rac1
ora....N1.lsnr ora....er.type ONLINE ONLINE rac1
ora.VOL.dg ora....up.type ONLINE ONLINE rac1
ora.asm ora.asm.type ONLINE ONLINE rac1
ora.cvu ora.cvu.type ONLINE ONLINE rac1
ora.gsd ora.gsd.type OFFLINE OFFLINE
ora....network ora....rk.type ONLINE ONLINE rac1
ora.oc4j ora.oc4j.type ONLINE ONLINE rac1
ora.ons ora.ons.type ONLINE ONLINE rac1
ora.pdap.db ora....se.type ONLINE ONLINE rac1
ora....SM1.asm application ONLINE ONLINE rac1
ora....C1.lsnr application ONLINE ONLINE rac1
ora.rac1.gsd application OFFLINE OFFLINE
ora.rac1.ons application ONLINE ONLINE rac1
ora.rac1.vip ora....t1.type ONLINE ONLINE rac1
ora.rac2.vip ora....t1.type ONLINE ONLINE rac1
ora....ry.acfs ora....fs.type ONLINE ONLINE rac1
ora.scan1.vip ora....ip.type ONLINE ONLINE rac1

2、模拟故障

2.1、删除安装目录

[root@rac2 u01]# rm -rf /u01 删除/u01

[root@rac2 u01]# ps -ef |grep ora_
root 12707 5522 0 08:48 pts/1 00:00:00 grep ora_数据库宕机

[root@rac2 u01]# ps -ef |grep asm asm进程还存在
grid 4754 1 0 08:43 ? 00:00:00 asm_pmon_+ASM2
grid 4756 1 0 08:43 ? 00:00:00 asm_psp0_+ASM2
grid 4758 1 1 08:43 ? 00:00:05 asm_vktm_+ASM2
grid 4762 1 0 08:43 ? 00:00:00 asm_gen0_+ASM2
grid 4764 1 0 08:43 ? 00:00:00 asm_diag_+ASM2
grid 4766 1 0 08:43 ? 00:00:00 asm_ping_+ASM2
grid 4768 1 0 08:43 ? 00:00:01 asm_dia0_+ASM2
grid 4770 1 0 08:43 ? 00:00:00 asm_lmon_+ASM2
grid 4772 1 0 08:43 ? 00:00:00 asm_lmd0_+ASM2
grid 4774 1 0 08:43 ? 00:00:01 asm_lms0_+ASM2
grid 4778 1 0 08:43 ? 00:00:00 asm_lmhb_+ASM2
grid 4780 1 0 08:43 ? 00:00:00 asm_mman_+ASM2
grid 4782 1 0 08:43 ? 00:00:00 asm_dbw0_+ASM2
grid 4784 1 0 08:43 ? 00:00:00 asm_lgwr_+ASM2
grid 4786 1 0 08:43 ? 00:00:00 asm_ckpt_+ASM2
grid 4788 1 0 08:43 ? 00:00:00 asm_smon_+ASM2
grid 4790 1 0 08:43 ? 00:00:00 asm_rbal_+ASM2
grid 4792 1 0 08:43 ? 00:00:00 asm_gmon_+ASM2
grid 4794 1 0 08:43 ? 00:00:00 asm_mmon_+ASM2
grid 4796 1 0 08:43 ? 00:00:00 asm_mmnl_+ASM2
grid 4798 1 0 08:43 ? 00:00:00 asm_lck0_+ASM2
grid 4833 1 0 08:43 ? 00:00:00 asm_asmb_+ASM2
grid 4835 1 0 08:43 ? 00:00:00 oracle+ASM2_asmb_+asm2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))
grid 4837 1 0 08:43 ? 00:00:00 asm_o000_+ASM2
grid 4839 1 0 08:43 ? 00:00:00 oracle+ASM2_o000_+asm2 (DESCRIPTION=(LOCAL=YES)(ADDRESS=(PROTOCOL=beq)))

[root@rac2 u01]# reboot 重启

[root@rac2 ~]# ps -ef |grep asm
root 3923 3890 0 09:00 pts/1 00:00:00 grep asmasm进程丢失。节点软件彻底损坏。

[root@rac2 ~]# mkdir -p /u01/11.2.0/grid 创建gird用户下的ORACLE_HOME

chown -R grid:oinstall /u01/11.2.0/grid

2.2、检查节点状态

[grid@rac1 ~]$ olsnodes -i
rac1 rac1-vip
rac2 rac2-vip
[grid@rac1 ~]$ olsnodes -n
rac1 1
rac2 2
[grid@rac1 ~]$ olsnodes -t
rac1 Unpinned
rac2 Unpinned
[grid@rac1 ~]$ olsnodes -s
rac1 Active
rac2 Inactive

2.3、查看实例信息

SQL> select thread#,status,instance from v$thread;

THREAD# STATUS INSTANCE
---------- ------ --------------------
1 OPEN pdap1

3、开始恢复

恢复步骤主要是
在RAC1上删除RAC2的节点信息

同步GRID的数据

同步database 数据

删除rac2的instance遗留信息

添加rac2的instance信息

3.1、在RAC1上删除RAC2的节点信息

root$oifcfg delif -node rac2

grid$ ./runInstaller -updateNodeList ORACLE_HOME=/u01/11.2.0/grid/ "CLUSTER_NODES={rac1}" CRS=TRUE

3.2、同步GRID的数据

export IGNORE_PREADDNODE_CHECKS=Y

grid$ ./addNode.sh -silent "CLUSTER_NEW_NODES={rac2}" "CLUSTER_NEW_VIRTUAL_HOSTNAMES={rac2-vip}" "CLUSTER_NEW_PRIVATE_NODE_NAMES={rac2-priv}" &>~/add_node.log

在新节点的root用户下执行脚本
/u01/app/oraInventory/orainstRoot.sh #On nodes racnode3
/u01/app/11.2.0/grid/root.sh #On nodes racnode3
To execute the configuration scripts:
1. Open a terminal window
2. Log in as "root"
3. Run the scripts in each cluster node

[root@racnode3 orasoft]# /u01/app/oraInventory/orainstRoot.sh
[root@racnode3 orasoft]# /u01/app/11.2.0/grid/root.sh

3.3、同步database 数据

3.3.1、自动同步

./addNode.sh CLUSTER_NEW_NODES={rac2} CLUSTER_NEW_VIRTUAL_HOSTNAMES={rac2-vip}

这里总是会失败

./addNode.sh CLUSTER_NEW_NODES={rac2} CLUSTER_NEW_VIRTUAL_HOSTNAMES={rac2-vip}

Performing pre-checks for node addition

Checking node reachability...

Node reachability check passed from node "rac1"

Checking user equivalence...

User equivalence check passed for user "oracle"

WARNING:

Node "rac2" already appears to be part of cluster

Pre-check for node addition was successful.

Starting Oracle Universal Installer...

Checking swap space: must be greater than 500 MB. Actual 3986 MB Passed

Oracle Universal Installer, Version 11.2.0.4.0 Production

Performing tests to see whether nodes rac2 are available

............................................................... 100% Done.

Error ocurred while retrieving node numbers of the existing nodes. Please check if clusterware home is properly configured.

SEVERE:Error ocurred while retrieving node numbers of the existing nodes. Please check if clusterware home is properly configured.

3.3.2、使用手工方法同步

首先准备好$ORACLE_HOME（oracle权限）和$ORACLE_BASE（同rac1一样的权限）和$ORACLE_BASE\admin（同rac权限）

压缩rac1上的oracle_home

rac1_oracle@$ tar -cvf 11g.tar db_1

传送至rac2的oracle_home

rac1_oracle@$scp 11g.tar rac2:/u01/app/oracle/product/11.2.0/

rac2上解压至oracle_home

rac2_oracle@$tar -xcvf 11g.tar

执行脚本

rac2_root@$./root.sh

3.4、修复实例

3.4、1删除rac2的instance遗留信息

rac1_root@$dbca

不多说，反正就是删除rac2的instance信息，小心点小是了。别删错了。

3.4.2、添加rac2的instance信息

（出现如下信息，代表就已经成功的修复/添加了一个节点）

[oracle@rac1 ~]$ dbca -silent -addInstance -nodeList rac2 -gdbName pdap -instanceName pdap2 -sysDBAUserName sys -sysDBAPassword "oracle"
Adding instance
1% complete
2% complete
6% complete
13% complete
20% complete
26% complete
33% complete
40% complete
46% complete
53% complete
66% complete
Completing instance management.
76% complete

100% complete

[grid@rac2 ~]$ crsctl status resource -t
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.DATA.dg
ONLINE ONLINE rac1
ONLINE ONLINE rac2
ora.LISTENER.lsnr
ONLINE ONLINE rac1
ONLINE ONLINE rac2
ora.VOL.dg
ONLINE ONLINE rac1
ONLINE ONLINE rac2
ora.asm
ONLINE ONLINE rac1 Started
ONLINE ONLINE rac2 Started
ora.gsd
OFFLINE OFFLINE rac1
OFFLINE OFFLINE rac2
ora.net1.network
ONLINE ONLINE rac1
ONLINE ONLINE rac2
ora.ons
ONLINE ONLINE rac1
ONLINE ONLINE rac2
ora.registry.acfs
ONLINE ONLINE rac1
ONLINE ONLINE rac2
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE rac1
ora.cvu
1 ONLINE ONLINE rac2
ora.oc4j
1 ONLINE ONLINE rac1
ora.pdap.db
1 ONLINE ONLINE rac1 Open
2 ONLINE ONLINE rac2 Open
ora.rac1.vip
1 ONLINE ONLINE rac1
ora.rac2.vip
1 ONLINE ONLINE rac2
ora.scan1.vip
1 ONLINE ONLINE rac1

oracle@rac2 ~]$ ps -ef |grep ora_
oracle 15737 1 0 13:27 ? 00:00:00 ora_pmon_pdap2
oracle 15739 1 0 13:27 ? 00:00:00 ora_psp0_pdap2
oracle 15756 1 1 13:27 ? 00:00:02 ora_vktm_pdap2
oracle 15760 1 0 13:27 ? 00:00:00 ora_gen0_pdap2
oracle 15762 1 0 13:27 ? 00:00:00 ora_diag_pdap2
oracle 15764 1 0 13:27 ? 00:00:00 ora_dbrm_pdap2
oracle 15766 1 0 13:27 ? 00:00:00 ora_ping_pdap2
oracle 15768 1 0 13:27 ? 00:00:00 ora_acms_pdap2
oracle 15770 1 0 13:27 ? 00:00:00 ora_dia0_pdap2
oracle 15772 1 0 13:27 ? 00:00:00 ora_lmon_pdap2
oracle 15774 1 0 13:27 ? 00:00:00 ora_lmd0_pdap2
oracle 15776 1 0 13:27 ? 00:00:00 ora_lms0_pdap2
oracle 15780 1 0 13:27 ? 00:00:00 ora_lms1_pdap2
oracle 15784 1 0 13:27 ? 00:00:00 ora_rms0_pdap2
oracle 15786 1 0 13:27 ? 00:00:00 ora_lmhb_pdap2
oracle 15788 1 0 13:27 ? 00:00:00 ora_mman_pdap2
oracle 15790 1 0 13:27 ? 00:00:00 ora_dbw0_pdap2
oracle 15792 1 0 13:27 ? 00:00:00 ora_lgwr_pdap2
oracle 15794 1 0 13:27 ? 00:00:00 ora_ckpt_pdap2
oracle 15796 1 0 13:27 ? 00:00:00 ora_smon_pdap2
oracle 15798 1 0 13:27 ? 00:00:00 ora_reco_pdap2
oracle 15800 1 0 13:27 ? 00:00:00 ora_rbal_pdap2
oracle 15802 1 0 13:27 ? 00:00:00 ora_asmb_pdap2
oracle 15804 1 0 13:27 ? 00:00:01 ora_mmon_pdap2
oracle 15808 1 0 13:27 ? 00:00:00 ora_mmnl_pdap2
oracle 15810 1 0 13:27 ? 00:00:00 ora_d000_pdap2
oracle 15812 1 0 13:27 ? 00:00:00 ora_s000_pdap2
oracle 15814 1 0 13:27 ? 00:00:00 ora_mark_pdap2
oracle 15816 1 0 13:27 ? 00:00:00 ora_ocf0_pdap2
oracle 15820 1 0 13:27 ? 00:00:00 ora_lck0_pdap2
oracle 15822 1 0 13:27 ? 00:00:00 ora_rsmn_pdap2
oracle 15842 1 0 13:27 ? 00:00:00 ora_o000_pdap2
oracle 15849 1 0 13:27 ? 00:00:00 ora_o001_pdap2
oracle 15855 1 0 13:27 ? 00:00:00 ora_gtx0_pdap2
oracle 15857 1 0 13:27 ? 00:00:00 ora_rcbg_pdap2
oracle 15859 1 0 13:27 ? 00:00:00 ora_qmnc_pdap2
oracle 15879 1 0 13:27 ? 00:00:00 ora_pz99_pdap2
oracle 15891 1 0 13:27 ? 00:00:00 ora_cjq0_pdap2
oracle 15894 1 0 13:27 ? 00:00:00 ora_q000_pdap2
oracle 15896 1 0 13:27 ? 00:00:00 ora_q001_pdap2

0 0