oracle11g rac 集群数据库节点能不正启动！

今天因地铁机房停电原因导致一数据库机器系统不能启动，开机启动报错如下：

网上查询解决命令：

fsck -y /dev/mapper/vg_dbserver1-lv_root

重点来了.数据库节点不能启动。启动出错是如下：

CRS-0184: Cannot communicate with the CRS daemon

数据库集群crsd.log 无明显报错！节点1脱离集群。最终发现节点1重启后时间为2015年。更改系统时间并同步系统时钟。处理完成后数据库正常启动。检查处理如下！

2018/5/9 18:46:38

[root@dbserver1 bin]# crsctl check has

CRS-4638: Oracle High Availability Services is online

[root@dbserver2 ~]# crsctl check crs

CRS-4638: Oracle High Availability Services is online

CRS-4537: Cluster Ready Services is online

CRS-4529: Cluster Synchronization Services is online

CRS-4533: Event Manager is online

[root@dbserver2 ~]# crs_stat -t

Name Type Target State Host

------------------------------------------------------------

ora.ARCH.dg ora....up.type ONLINE ONLINE dbserver2

ora.DATA.dg ora....up.type ONLINE ONLINE dbserver2

ora.DG1.dg ora....up.type ONLINE ONLINE dbserver2

ora.DG2.dg ora....up.type ONLINE ONLINE dbserver2

ora.DG3.dg ora....up.type ONLINE ONLINE dbserver2

ora....ER.lsnr ora....er.type ONLINE ONLINE dbserver2

ora....N1.lsnr ora....er.type ONLINE ONLINE dbserver2

ora.RCY1.dg ora....up.type ONLINE ONLINE dbserver2

ora....er1.vip ora....t1.type ONLINE ONLINE dbserver2

ora....SM2.asm application ONLINE ONLINE dbserver2

ora....R2.lsnr application ONLINE ONLINE dbserver2

ora....er2.gsd application OFFLINE OFFLINE

ora....er2.ons application ONLINE ONLINE dbserver2

ora....er2.vip ora....t1.type ONLINE ONLINE dbserver2

ora....network ora....rk.type ONLINE ONLINE dbserver2

ora....flow.db ora....se.type ONLINE ONLINE dbserver2

[root@dbserver1 bin]# ifconfig -a

eth0 Link encap:Ethernet HWaddr 00:25:90:5D:A8:46

inet addr:172.168.100.5 Bcast:172.168.100.255 Mask:255.255.255.0

inet6 addr: fe80::225:90ff:fe5d:a846/64 Scope:Link

UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1

RX packets:4103 errors:0 dropped:0 overruns:0 frame:0

TX packets:2937 errors:0 dropped:0 overruns:0 carrier:0

collisions:0 txqueuelen:1000

RX bytes:503937 (492.1 KiB) TX bytes:441698 (431.3 KiB)

Memory:dfe20000-dfe3ffff

eth1 Link encap:Ethernet HWaddr 00:25:90:5D:A8:47

BROADCAST MULTICAST MTU:1500 Metric:1

RX packets:0 errors:0 dropped:0 overruns:0 frame:0

TX packets:0 errors:0 dropped:0 overruns:0 carrier:0

collisions:0 txqueuelen:1000

RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)

Memory:dfe60000-dfe7ffff

eth2 Link encap:Ethernet HWaddr 00:25:90:5D:A8:48

inet addr:10.10.10.1 Bcast:10.10.10.255 Mask:255.255.255.0

inet6 addr: fe80::225:90ff:fe5d:a848/64 Scope:Link

UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1

RX packets:64646 errors:0 dropped:0 overruns:0 frame:0

TX packets:25629 errors:0 dropped:0 overruns:0 carrier:0

collisions:0 txqueuelen:1000

Memory:dfea0000-dfebffff

eth2:1 Link encap:Ethernet HWaddr 00:25:90:5D:A8:48

inet addr:169.254.231.5 Bcast:169.254.255.255 Mask:255.255.0.0

UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1

Memory:dfea0000-dfebffff

eth3 Link encap:Ethernet HWaddr 00:25:90:5D:A8:49

BROADCAST MULTICAST MTU:1500 Metric:1

RX packets:0 errors:0 dropped:0 overruns:0 frame:0

TX packets:0 errors:0 dropped:0 overruns:0 carrier:0

collisions:0 txqueuelen:1000

RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)

Memory:dfee0000-dfefffff

lo Link encap:Local Loopback

inet addr:127.0.0.1 Mask:255.0.0.0

inet6 addr: ::1/128 Scope:Host

UP LOOPBACK RUNNING MTU:65536 Metric:1

RX packets:2474 errors:0 dropped:0 overruns:0 frame:0

TX packets:2474 errors:0 dropped:0 overruns:0 carrier:0

collisions:0 txqueuelen:0

RX bytes:461462 (450.6 KiB) TX bytes:461462 (450.6 KiB)

[root@dbserver1 bin]#

[root@dbserver2 ~]# ifconfig -a

eth0 Link encap:Ethernet HWaddr 00:25:90:5D:D2:62

inet addr:172.168.100.6 Bcast:172.168.100.255 Mask:255.255.255.0

inet6 addr: fe80::225:90ff:fe5d:d262/64 Scope:Link

UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1

RX packets:1231494 errors:0 dropped:0 overruns:0 frame:0

TX packets:1162628 errors:0 dropped:0 overruns:0 carrier:0

collisions:0 txqueuelen:1000

Memory:dfe20000-dfe3ffff

eth0:1 Link encap:Ethernet HWaddr 00:25:90:5D:D2:62

inet addr:172.168.100.9 Bcast:172.168.100.255 Mask:255.255.255.0

UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1

Memory:dfe20000-dfe3ffff

eth0:2 Link encap:Ethernet HWaddr 00:25:90:5D:D2:62

inet addr:172.168.100.7 Bcast:172.168.100.255 Mask:255.255.255.0

UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1

Memory:dfe20000-dfe3ffff

eth0:3 Link encap:Ethernet HWaddr 00:25:90:5D:D2:62

inet addr:172.168.100.8 Bcast:172.168.100.255 Mask:255.255.255.0

UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1

Memory:dfe20000-dfe3ffff

eth1 Link encap:Ethernet HWaddr 00:25:90:5D:D2:63

BROADCAST MULTICAST MTU:1500 Metric:1

RX packets:0 errors:0 dropped:0 overruns:0 frame:0

TX packets:0 errors:0 dropped:0 overruns:0 carrier:0

collisions:0 txqueuelen:1000

RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)

Memory:dfe60000-dfe7ffff

eth2 Link encap:Ethernet HWaddr 00:25:90:5D:D2:64

inet addr:10.10.10.2 Bcast:10.10.10.255 Mask:255.255.255.0

inet6 addr: fe80::225:90ff:fe5d:d264/64 Scope:Link

UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1

RX packets:83825 errors:0 dropped:0 overruns:0 frame:0

TX packets:200633 errors:0 dropped:0 overruns:0 carrier:0

collisions:0 txqueuelen:1000

Memory:dfea0000-dfebffff

eth2:1 Link encap:Ethernet HWaddr 00:25:90:5D:D2:64

inet addr:169.254.59.187 Bcast:169.254.255.255 Mask:255.255.0.0

UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1

Memory:dfea0000-dfebffff

eth3 Link encap:Ethernet HWaddr 00:25:90:5D:D2:65

BROADCAST MULTICAST MTU:1500 Metric:1

RX packets:0 errors:0 dropped:0 overruns:0 frame:0

TX packets:0 errors:0 dropped:0 overruns:0 carrier:0

collisions:0 txqueuelen:1000

RX bytes:0 (0.0 b) TX bytes:0 (0.0 b)

Memory:dfee0000-dfefffff

lo Link encap:Local Loopback

inet addr:127.0.0.1 Mask:255.0.0.0

inet6 addr: ::1/128 Scope:Host

UP LOOPBACK RUNNING MTU:65536 Metric:1

RX packets:61389 errors:0 dropped:0 overruns:0 frame:0

TX packets:61389 errors:0 dropped:0 overruns:0 carrier:0

collisions:0 txqueuelen:0

[root@dbserver2 ~]#

[root@dbserver1 bin]# date

Wed Jun 24 11:57:02 CST 2015

[root@dbserver1 bin]#

[root@dbserver2 ~]# date

Wed May 9 18:13:15 CST 2018

[root@dbserver1 bin]# date

Wed Jun 24 12:05:42 CST 2015

[root@dbserver1 bin]# date -s 05/09/2018

Wed May 9 00:00:00 CST 2018

[root@dbserver1 bin]# date

Wed May 9 00:00:09 CST 2018

[root@dbserver1 bin]# date -s 18:22:50

Wed May 9 18:22:50 CST 2018

[root@dbserver1 bin]# date

Wed May 9 18:22:54 CST 2018

[root@dbserver1 bin]#hwclock --systohc

[root@dbserver1 ~]# crs_stat -t

Name Type Target State Host

------------------------------------------------------------

ora.ARCH.dg ora....up.type ONLINE ONLINE dbserver1

ora.DATA.dg ora....up.type ONLINE ONLINE dbserver1

ora.DG1.dg ora....up.type ONLINE ONLINE dbserver1

ora.DG2.dg ora....up.type ONLINE ONLINE dbserver1

ora.DG3.dg ora....up.type ONLINE ONLINE dbserver1

ora....ER.lsnr ora....er.type ONLINE ONLINE dbserver1

ora....N1.lsnr ora....er.type ONLINE ONLINE dbserver2

ora.RCY1.dg ora....up.type ONLINE ONLINE dbserver1

ora....SM1.asm application ONLINE ONLINE dbserver1

ora....R1.lsnr application ONLINE ONLINE dbserver1

ora....er1.gsd application OFFLINE OFFLINE

ora....er1.ons application ONLINE ONLINE dbserver1

ora....er1.vip ora....t1.type ONLINE ONLINE dbserver1

ora....SM2.asm application ONLINE ONLINE dbserver2

ora....R2.lsnr application ONLINE ONLINE dbserver2

ora....er2.gsd application OFFLINE OFFLINE

ora....er2.ons application ONLINE ONLINE dbserver2

ora....er2.vip ora....t1.type ONLINE ONLINE dbserver2

ora....network ora....rk.type ONLINE ONLINE dbserver1

ora....flow.db ora....se.type ONLINE ONLINE dbserver1

[root@dbserver1 ~]#

SQL> select instance_name,status from gv$instance;

INSTANCE_NAMESTATUS

---------------- ------------

workflow1OPEN

workflow2OPEN