one rac troubleshooting

登陆节点rac1 后发现：

 

[root@rac1 ~]# su - oracle
rac1-> sqlplus /nolog

SQL*Plus: Release 10.2.0.1.0 - Production on Mon Mar 16 09:28:53 2009

Copyright (c) 1982, 2005, Oracle.  All rights reserved.

SQL> conn / as sysdba
Connected.
SQL> select group#,thread#,status,members from v$log order by 2,1;

    GROUP#    THREAD# STATUS              MEMBERS
---------- ---------- ---------------- ----------
         1          1 CURRENT                   1
         2          1 INACTIVE                  1
         5          1 INACTIVE                  2
         3          2 INACTIVE                  1
         4          2 CURRENT                   1
         7          2 INACTIVE                  2

6 rows selected.

SQL> select * from v$logfile order by 1;

    GROUP# STATUS  TYPE    MEMBER                                   IS_
---------- ------- ------- ---------------------------------------- ---
         1         ONLINE  /diskc/oradata/demo/redo01.log           NO
         2         ONLINE  /diskc/oradata/demo/redo02.log           NO
         3 STALE   ONLINE  /diskc/oradata/demo/redo03.log           NO
         4         ONLINE  /diskc/oradata/demo/redo04.log           NO
         5         ONLINE  /diskc/oradata/demo/redo05_01.log        NO
         5         ONLINE  /diskc/oradata/demo/redo05_02.log        NO
         7 STALE   ONLINE  /diskc/oradata/demo/redo07_01.log        NO
         7 STALE   ONLINE  /diskc/oradata/demo/redo07_02.log        NO

8 rows selected.

 

登陆节点rac2：

 

[root@rac2 ~]# su - oracle
rac2-> sqlplus /nolog

SQL*Plus: Release 10.2.0.1.0 - Production on Mon Mar 16 09:40:51 2009

Copyright (c) 1982, 2005, Oracle.  All rights reserved.

SQL> conn / as sysdba 
Connected to an idle instance.
SQL>

这一个节点上的rac竟然没有启动！！！

开始troubleshooting，不懂rac啊。。。

rac1-> crs_stat -t
Name           Type           Target    State     Host       
------------------------------------------------------------
ora.demo.db    application    ONLINE    ONLINE    rac1       
ora....o1.inst application    ONLINE    ONLINE    rac1       
ora....o2.inst application    ONLINE    OFFLINE              
ora....C1.lsnr application    ONLINE    ONLINE    rac1       
ora.rac1.gsd   application    ONLINE    ONLINE    rac1       
ora.rac1.ons   application    ONLINE    ONLINE    rac1       
ora.rac1.vip   application    ONLINE    ONLINE    rac1       
ora....C2.lsnr application    ONLINE    OFFLINE              
ora.rac2.gsd   application    ONLINE    OFFLINE              
ora.rac2.ons   application    ONLINE    OFFLINE              
ora.rac2.vip   application    ONLINE    ONLINE    rac1  

[root@rac2 bin]# ./crsctl start crs
Attempting to start CRS stack
The CRS stack will be started shortly
[root@rac2 bin]# ./crsctl start resources
Starting resources.
Successfully started CRS resources
[root@rac2 bin]# ./crsctl start ora.rac2.ons
Unexpected parameter: ora.rac2.ons
[root@rac2 bin]#

 

[root@rac2 ~]# su - oracle
rac2-> srvctl start nodeapps -n rac2
CRS-1028: Dependency analysis failed because of:
CRS-0223: Resource 'ora.rac2.gsd' has placement error.
CRS-1028: Dependency analysis failed because of:
CRS-0223: Resource 'ora.rac2.ons' has placement error.
CRS-1028: Dependency analysis failed because of:
CRS-0223: Resource 'ora.rac2.LISTENER_RAC2.lsnr' has placement error.
rac2->

rac2-> crsctl check crs
CSS appears healthy
CRS appears healthy
EVM appears healthy

rac2-> crsctl start resources
Starting resources.
Successfully started CRS resources
rac2-> crs_stat -t
Name           Type           Target    State     Host       
------------------------------------------------------------
ora.demo.db    application    ONLINE    ONLINE    rac1       
ora....o1.inst application    ONLINE    ONLINE    rac1       
ora....o2.inst application    ONLINE    UNKNOWN   rac2       
ora....C1.lsnr application    ONLINE    ONLINE    rac1       
ora.rac1.gsd   application    ONLINE    ONLINE    rac1       
ora.rac1.ons   application    ONLINE    ONLINE    rac1       
ora.rac1.vip   application    ONLINE    ONLINE    rac1       
ora....C2.lsnr application    ONLINE    UNKNOWN   rac2       
ora.rac2.gsd   application    ONLINE    UNKNOWN   rac2       
ora.rac2.ons   application    ONLINE    UNKNOWN   rac2       
ora.rac2.vip   application    ONLINE    ONLINE    rac2       
rac2-> pwd
/u01/oracle/
rac2-> cd $ORA_CRS_HOME/bin
rac2-> pwd
/u01/app/oracle/product/10.2.0/crs_1/bin
rac2-> gsdctl stop
rac2-> gsdctl status
rac2-> ./gsdctl status
rac2-> gsdctl start
rac2-> srvctl start nodeapps -n rac2
CRS-1028: Dependency analysis failed because of:
CRS-0223: Resource 'ora.rac2.gsd' has placement error.
CRS-1028: Dependency analysis failed because of:
CRS-0223: Resource 'ora.rac2.ons' has placement error.
CRS-1028: Dependency analysis failed because of:
CRS-0223: Resource 'ora.rac2.LISTENER_RAC2.lsnr' has placement error.
rac2-> exec /usr/bin/ssh-agent $SHELL
rac2-> /usr/bin/ssh-add
Enter passphrase for /u01/oracle//.ssh/id_rsa:
Identity added: /u01/oracle//.ssh/id_rsa (/u01/oracle//.ssh/id_rsa)
Identity added: /u01/oracle//.ssh/id_dsa (/u01/oracle//.ssh/id_dsa)
rac2-> ssh rac1 date
Mon Mar 16 09:54:42 CST 2009
rac2-> ssh rac2 date
Mon Mar 16 09:54:50 CST 2009

两个节点上的时间同步的啊。。。

rac2-> crs_stop -all
Attempting to stop `ora.rac1.gsd` on member `rac1`
Stop of `ora.rac1.gsd` on member `rac1` succeeded.
Attempting to stop `ora.rac1.ons` on member `rac1`
Stop of `ora.rac1.ons` on member `rac1` succeeded.
Attempting to stop `ora.demo.db` on member `rac1`
Stop of `ora.demo.db` on member `rac1` succeeded.
Attempting to stop `ora.rac1.LISTENER_RAC1.lsnr` on member `rac1`
Attempting to stop `ora.rac2.LISTENER_RAC2.lsnr` on member `rac2`
Stop of `ora.rac1.LISTENER_RAC1.lsnr` on member `rac1` succeeded.
Stop of `ora.rac2.LISTENER_RAC2.lsnr` on member `rac2` succeeded.
Attempting to stop `ora.rac2.vip` on member `rac2`
`ora.demo.demo1.inst` is already OFFLINE.
Stop of `ora.rac2.vip` on member `rac2` succeeded.
Attempting to stop `ora.rac1.vip` on member `rac1`
Stop of `ora.rac1.vip` on member `rac1` succeeded.
CRS-0216: Could not stop resource 'ora.demo.demo1.inst'.

rac2-> crs_stat -t
Name           Type           Target    State     Host       
------------------------------------------------------------
ora.demo.db    application    OFFLINE   OFFLINE              
ora....o1.inst application    OFFLINE   OFFLINE              
ora....o2.inst application    OFFLINE   OFFLINE              
ora....C1.lsnr application    OFFLINE   OFFLINE              
ora.rac1.gsd   application    OFFLINE   OFFLINE              
ora.rac1.ons   application    OFFLINE   OFFLINE              
ora.rac1.vip   application    OFFLINE   OFFLINE              
ora....C2.lsnr application    OFFLINE   OFFLINE              
ora.rac2.gsd   application    ONLINE    UNKNOWN   rac2       
ora.rac2.ons   application    ONLINE    UNKNOWN   rac2       
ora.rac2.vip   application    OFFLINE   OFFLINE              
rac2-> crs_stop -all
rac2->
rac2-> crs_stat
NAME=ora.demo.db
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.demo.demo1.inst
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.demo.demo2.inst
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.rac1.LISTENER_RAC1.lsnr
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.rac1.gsd
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.rac1.ons
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.rac1.vip
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.rac2.LISTENER_RAC2.lsnr
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

NAME=ora.rac2.gsd
TYPE=application
TARGET=ONLINE
STATE=UNKNOWN on rac2

NAME=ora.rac2.ons
TYPE=application
TARGET=ONLINE
STATE=UNKNOWN on rac2

NAME=ora.rac2.vip
TYPE=application
TARGET=OFFLINE
STATE=OFFLINE

rac2-> crs_stat -t
Name           Type           Target    State     Host       
------------------------------------------------------------
ora.demo.db    application    OFFLINE   OFFLINE              
ora....o1.inst application    OFFLINE   OFFLINE              
ora....o2.inst application    OFFLINE   OFFLINE              
ora....C1.lsnr application    OFFLINE   OFFLINE              
ora.rac1.gsd   application    OFFLINE   OFFLINE              
ora.rac1.ons   application    OFFLINE   OFFLINE              
ora.rac1.vip   application    OFFLINE   OFFLINE              
ora....C2.lsnr application    OFFLINE   OFFLINE              
ora.rac2.gsd   application    ONLINE    UNKNOWN   rac2       
ora.rac2.ons   application    ONLINE    UNKNOWN   rac2       
ora.rac2.vip   application    OFFLINE   OFFLINE              
rac2-> crs_stop ora.rac2.gsd
Attempting to stop `ora.rac2.gsd` on member `rac2`
Stop of `ora.rac2.gsd` on member `rac2` succeeded.
rac2-> crs_stop ora.rac2.ons
Attempting to stop `ora.rac2.ons` on member `rac2`
Stop of `ora.rac2.ons` on member `rac2` succeeded.
rac2-> crs_stat -t
Name           Type           Target    State     Host       
------------------------------------------------------------
ora.demo.db    application    OFFLINE   OFFLINE              
ora....o1.inst application    OFFLINE   OFFLINE              
ora....o2.inst application    OFFLINE   OFFLINE              
ora....C1.lsnr application    OFFLINE   OFFLINE              
ora.rac1.gsd   application    OFFLINE   OFFLINE              
ora.rac1.ons   application    OFFLINE   OFFLINE              
ora.rac1.vip   application    OFFLINE   OFFLINE              
ora....C2.lsnr application    OFFLINE   OFFLINE              
ora.rac2.gsd   application    OFFLINE   OFFLINE              
ora.rac2.ons   application    OFFLINE   OFFLINE              
ora.rac2.vip   application    OFFLINE   OFFLINE              
rac2-> crs_start -all
Attempting to start `ora.rac1.vip` on member `rac1`
Attempting to start `ora.rac2.vip` on member `rac2`
Start of `ora.rac1.vip` on member `rac1` succeeded.
Attempting to start `ora.demo.demo1.inst` on member `rac1`
Start of `ora.rac2.vip` on member `rac2` succeeded.
Attempting to start `ora.demo.demo2.inst` on member `rac2`
Start of `ora.demo.demo1.inst` on member `rac1` succeeded.
Attempting to start `ora.rac1.LISTENER_RAC1.lsnr` on member `rac1`
Start of `ora.rac1.LISTENER_RAC1.lsnr` on member `rac1` succeeded.
Start of `ora.demo.demo2.inst` on member `rac2` succeeded.
Attempting to start `ora.rac2.LISTENER_RAC2.lsnr` on member `rac2`
Start of `ora.rac2.LISTENER_RAC2.lsnr` on member `rac2` succeeded.
CRS-1002: Resource 'ora.rac1.ons' is already running on member 'rac1'

Attempting to start `ora.rac1.gsd` on member `rac1`
Start of `ora.rac1.gsd` on member `rac1` succeeded.
CRS-1002: Resource 'ora.rac2.ons' is already running on member 'rac2'

Attempting to start `ora.rac2.gsd` on member `rac2`
Attempting to start `ora.demo.db` on member `rac2`
Start of `ora.rac2.gsd` on member `rac2` succeeded.
Start of `ora.demo.db` on member `rac2` succeeded.
CRS-0223: Resource 'ora.rac1.ons' has placement error.

CRS-0223: Resource 'ora.rac2.ons' has placement error.

rac2->
rac2-> crs_stat -t
Name           Type           Target    State     Host       
------------------------------------------------------------
ora.demo.db    application    ONLINE    ONLINE    rac2       
ora....o1.inst application    ONLINE    ONLINE    rac1       
ora....o2.inst application    ONLINE    ONLINE    rac2       
ora....C1.lsnr application    ONLINE    ONLINE    rac1       
ora.rac1.gsd   application    ONLINE    ONLINE    rac1       
ora.rac1.ons   application    ONLINE    ONLINE    rac1       
ora.rac1.vip   application    ONLINE    ONLINE    rac1       
ora....C2.lsnr application    ONLINE    ONLINE    rac2       
ora.rac2.gsd   application    ONLINE    ONLINE    rac2       
ora.rac2.ons   application    ONLINE    ONLINE    rac2       
ora.rac2.vip   application    ONLINE    ONLINE    rac2       
rac2-> 

SQL> conn / as sysdba
Connected.
SQL> select instance_name,status from v$instance;

INSTANCE_NAME    STATUS
---------------- ------------
demo2            OPEN

SQL> select name,open_mode ,log_mode from v$database;

NAME      OPEN_MODE  LOG_MODE
--------- ---------- ------------
DEMO      READ WRITE ARCHIVELOG

SQL> alter system switch logfile;

System altered.

SQL> col member for a40
SQL> select * from v$logfile;      

    GROUP# STATUS  TYPE    MEMBER                                   IS_
---------- ------- ------- ---------------------------------------- ---
         2         ONLINE  /diskc/oradata/demo/redo02.log           NO
         1         ONLINE  /diskc/oradata/demo/redo01.log           NO
         3 STALE   ONLINE  /diskc/oradata/demo/redo03.log           NO
         4         ONLINE  /diskc/oradata/demo/redo04.log           NO
         7         ONLINE  /diskc/oradata/demo/redo07_01.log        NO
         7         ONLINE  /diskc/oradata/demo/redo07_02.log        NO
         5         ONLINE  /diskc/oradata/demo/redo05_01.log        NO
         5         ONLINE  /diskc/oradata/demo/redo05_02.log        NO

SQL> select group#,thread# from v$log order by 2,1;

    GROUP#    THREAD#
---------- ----------
         1          1
         2          1
         5          1
         3          2
         4          2
         7          2

6 rows selected.

SQL> alter system switch logfile;

System altered.

SQL> select * from v$logfile;

    GROUP# STATUS  TYPE    MEMBER                                   IS_
---------- ------- ------- ---------------------------------------- ---
         2         ONLINE  /diskc/oradata/demo/redo02.log           NO
         1         ONLINE  /diskc/oradata/demo/redo01.log           NO
         3         ONLINE  /diskc/oradata/demo/redo03.log           NO
         4         ONLINE  /diskc/oradata/demo/redo04.log           NO
         7         ONLINE  /diskc/oradata/demo/redo07_01.log        NO
         7         ONLINE  /diskc/oradata/demo/redo07_02.log        NO
         5         ONLINE  /diskc/oradata/demo/redo05_01.log        NO
         5         ONLINE  /diskc/oradata/demo/redo05_02.log        NO

8 rows selected.

SQL>