Oracle HAS startup failed, After stop syslogd

Just something strange or ..., Oracle High Availability Services startup failed. After stopped syslogd. Hope someone explain this issue.
Using 11gR2 Grid + ASM LIBS + Loop devices.

Let me show:

# /etc/init.d/syslog stop
Shutting down kernel logger: [ OK ]
Shutting down system logger: [ OK ]

# /oracle/grid2/bin/crsctl start has
CRS-4124: Oracle High Availability Services startup failed.
CRS-4000: Command Start failed, or completed with errors.

# /oracle/grid2/bin/crsctl check has
CRS-4639: Could not contact Oracle High Availability Services

# /etc/init.d/syslog start
Starting system logger: [ OK ]
Starting kernel logger: [ OK ]

# /oracle/grid2/bin/crsctl start has
CRS-4123: Oracle High Availability Services has been started.

# /oracle/grid2/bin/crsctl check has
CRS-4638: Oracle High Availability Services is online

and then Check more:
Case: syslogd stopped and HAS startup failed
# /etc/init.d/syslog status
syslogd is stopped
klogd is stopped

# strace /oracle/grid2/bin/crsctl start has
execve("/oracle/grid2/bin/crsctl", ["/oracle/grid2/bin/crsctl", "start", "has"], [/* 23 vars */]) = 0
brk(0) = 0xa684000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b1c9c71f000
uname({sys="Linux", node="RHEL5-TEST", ...}) = 0
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=116257, ...}) = 0
mmap(NULL, 116257, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2b1c9c72000
.
.
.
socket(PF_FILE, SOCK_STREAM, 0) = 4
connect(4, {sa_family=AF_FILE, path="/var/tmp/.oracle/sOHASD_UI_SOCKET"...}, 110) = -1 ECONNREFUSED (Connection refused)
access("/var/tmp/.oracle/sOHASD_UI_SOCKET", F_OK) = 0
close(4) = 0
close(3) = 0
open("/oracle/grid2/crs/mesg/crsus.msb", O_RDONLY) = 3
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
lseek(3, 0, SEEK_SET) = 0
read(3, "\25\23\"\1\23\3\t\t\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 256) = 256
lseek(3, 512, SEEK_SET) = 512
read(3, "\21'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 1024, SEEK_SET) = 1024
read(3, "\226\0\245\0\262\0\307\0\330\0\345\0\364\0\376\0\7\1\21\0013\1;\1D\1\231\1\303\1\370\1"..., 512) = 512
lseek(3, 99840, SEEK_SET) = 99840
read(3, "\t\0\37\22\0\0>\0 \22\0\0q\0!\22\0\0\244\0\"\22\0\0\313\0#\22\1\0\363\0"..., 512) = 512
lseek(3, 115712, SEEK_SET) = 115712
read(3, "\377\377\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 116224, SEEK_SET) = 116224
read(3, "\377\377\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 116736, SEEK_SET) = 116736
read(3, "\0\0\0\0\0\0\10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
close(3) = 0
nanosleep({5, 0}, {0, 422276128}) = 0
open("/oracle/grid2/crs/mesg/crsus.msb", O_RDONLY) = 3
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
lseek(3, 0, SEEK_SET) = 0
read(3, "\25\23\"\1\23\3\t\t\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 256) = 256
lseek(3, 512, SEEK_SET) = 512
read(3, "\21'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 1024, SEEK_SET) = 1024
read(3, "\226\0\245\0\262\0\307\0\330\0\345\0\364\0\376\0\7\1\21\0013\1;\1D\1\231\1\303\1\370\1"..., 512) = 512
lseek(3, 75264, SEEK_SET) = 75264
read(3, "\3\0\32\20\3\0\32\0\33\20\0\0\30\1\34\20\0\0K\1\0\0\0\0|\1 crsc"..., 512) = 512
lseek(3, 115712, SEEK_SET) = 115712
read(3, "\377\377\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 116224, SEEK_SET) = 116224
read(3, "\377\377\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 116736, SEEK_SET) = 116736
read(3, "\0\0\0\0\0\0\10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
close(3) = 0
fstat(1, {st_mode=S_IFREG|0644, st_size=249965, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaade2000
open("/oracle/grid2/crs/mesg/crsus.msb", O_RDONLY) = 3
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
lseek(3, 0, SEEK_SET) = 0
read(3, "\25\23\"\1\23\3\t\t\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 256) = 256
lseek(3, 512, SEEK_SET) = 512
read(3, "\21'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 1024, SEEK_SET) = 1024
read(3, "\226\0\245\0\262\0\307\0\330\0\345\0\364\0\376\0\7\1\21\0013\1;\1D\1\231\1\303\1\370\1"..., 512) = 512
lseek(3, 52224, SEEK_SET) = 52224
read(3, "\10\0\353\n\1\0008\0\354\n\1\0\212\0\355\n\3\0\320\0\356\n\0\0\24\1\240\17\0\0q\1"..., 512) = 512
lseek(3, 115712, SEEK_SET) = 115712
read(3, "\377\377\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 115712, SEEK_SET) = 115712
read(3, "\377\377\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 116224, SEEK_SET) = 116224
read(3, "\377\377\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 116736, SEEK_SET) = 116736
read(3, "\0\0\0\0\0\0\10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
close(3) = 0
write(1, "CRS-4124: Oracle High Availabili"..., 118CRS-4124: Oracle High Availability Services startup failed.
CRS-4000: Command Start failed, or completed with errors.
) = 118
munmap(0x2aaaaacd7000, 1091040) = 0
futex(0x19214b84, FUTEX_CMP_REQUEUE_PRIVATE, 1, 2147483647, 0x19214ab0, 4) = 1
futex(0x19214ab0, FUTEX_WAKE_PRIVATE, 1) = 1
brk(0x192d7000) = 0x192d7000
munmap(0x2b183c79f000, 4198400) = 0
munmap(0x2b183c77c000, 143360) = 0
munmap(0x2b183c759000, 143360) = 0
munmap(0x2b183c736000, 143360) = 0
exit_group(1) = ?

Case: syslogd start and HAS startup... -)
# /etc/init.d/syslog status
syslogd (pid 17175) is running...
klogd (pid 17178) is running...

# strace /oracle/grid2/bin/crsctl start has
execve("/oracle/grid2/bin/crsctl", ["/oracle/grid2/bin/crsctl", "start", "has"], [/* 23 vars */]) = 0
brk(0) = 0x17268000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b0b8bab9000
uname({sys="Linux", node="RHEL5-TEST", ...}) = 0
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=116257, ...}) = 0
mmap(NULL, 116257, PROT_READ, MAP_PRIVATE, 3, 0) = 0x2b0b8baba000
.
.
.
open("/oracle/grid2/auth/ohasd/rhel5-test/A5291056/8f9b98b1", O_WRONLY|O_CREAT|O_EXCL, 0644) = 5
fchmod(5, 0644) = 0
write(5, "c\200\n#", 4) = 4
close(5) = 0
write(4, "4\0\0\0\2\0\2\0\1\1\1\1\3\0\0\0\0\0\0\0\1\0\0\0\362\2161\1\0\0\0\0"..., 52) = 52
write(4, "8\0\0\0\3\0\2\0\1\1\1\1\4\0\0\0\0\0\0\0\1\0\0\0\362\2161\1\0\0\0\0"..., 56) = 56
times({tms_utime=2, tms_stime=2, tms_cutime=0, tms_cstime=0}) = 431499395
times({tms_utime=2, tms_stime=2, tms_cutime=0, tms_cstime=0}) = 431499395
poll([{fd=3, events=POLLIN|POLLRDNORM}, {fd=4, events=POLLIN|POLLRDNORM}], 2, -1) = 1 ([{fd=4, revents=POLLIN|POLLRDNORM}])
times({tms_utime=2, tms_stime=2, tms_cutime=0, tms_cstime=0}) = 431499395
times({tms_utime=2, tms_stime=2, tms_cutime=0, tms_cstime=0}) = 431499395
read(4, "8\0\0\0\3\0\2\0\1\1\1\1\3\0\0\0\0\0\0\0\1\0\0\0\36\2251\1\0\0\0\0"..., 32768) = 56
write(4, "\237\1\0\0PC\0\0\v\2\2\0\5\0\0\0\0\0\0\0\1\0\0\0\362\2161\1\0\0\0\0"..., 415) = 415
times({tms_utime=2, tms_stime=2, tms_cutime=0, tms_cstime=0}) = 431499395
poll([{fd=3, events=POLLIN|POLLRDNORM}, {fd=4, events=POLLIN|POLLRDNORM}], 2, -1) = 1 ([{fd=4, revents=POLLIN|POLLRDNORM}])
times({tms_utime=2, tms_stime=2, tms_cutime=0, tms_cstime=0}) = 431499500
times({tms_utime=2, tms_stime=2, tms_cutime=0, tms_cstime=0}) = 431499500
read(4, "\211\2\0\0PC\0\0\v\2\2\0\4\0\0\0\0\0\0\0\1\0\0\0\36\2251\1\0\0\0\0"..., 32768) = 649
times({tms_utime=2, tms_stime=2, tms_cutime=0, tms_cstime=0}) = 431499500
times({tms_utime=2, tms_stime=2, tms_cutime=0, tms_cstime=0}) = 431499500
times({tms_utime=2, tms_stime=2, tms_cutime=0, tms_cstime=0}) = 431499500
write(4, "0\0\0\0\2\0\20\0\1\0\1\0\0\0\0\0\0\0\0\0\0\0\0\0\362\2161\1\0\0\0\0"..., 48) = 48
fcntl(4, F_GETFL) = 0x802 (flags O_RDWR|O_NONBLOCK)
fcntl(4, F_SETFL, O_RDWR) = 0
close(4) = 0
close(3) = 0
open("/oracle/grid2/crs/mesg/crsus.msb", O_RDONLY) = 3
fcntl(3, F_SETFD, FD_CLOEXEC) = 0
lseek(3, 0, SEEK_SET) = 0
read(3, "\25\23\"\1\23\3\t\t\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 256) = 256
lseek(3, 512, SEEK_SET) = 512
read(3, "\21'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 1024, SEEK_SET) = 1024
read(3, "\226\0\245\0\262\0\307\0\330\0\345\0\364\0\376\0\7\1\21\0013\1;\1D\1\231\1\303\1\370\1"..., 512) = 512
lseek(3, 75264, SEEK_SET) = 75264
read(3, "\3\0\32\20\3\0\32\0\33\20\0\0\30\1\34\20\0\0K\1\0\0\0\0|\1 crsc"..., 512) = 512
lseek(3, 115712, SEEK_SET) = 115712
read(3, "\377\377\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 116224, SEEK_SET) = 116224
read(3, "\377\377\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
lseek(3, 116736, SEEK_SET) = 116736
read(3, "\0\0\0\0\0\0\10\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"..., 512) = 512
close(3) = 0
fstat(1, {st_mode=S_IFREG|0644, st_size=69208, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2aaaaade2000
write(1, "CRS-4123: Oracle High Availabili"..., 62CRS-4123: Oracle High Availability Services has been started.
) = 62
munmap(0x2aaaaacd7000, 1091040) = 0
futex(0xd178b84, FUTEX_CMP_REQUEUE_PRIVATE, 1, 2147483647, 0xd178ab0, 4) = 1
times({tms_utime=2, tms_stime=2, tms_cutime=0, tms_cstime=0}) = 431499500
nanosleep({0, 100000000}, {140734224759904, 4314995000}) = 0
times({tms_utime=2, tms_stime=2, tms_cutime=0, tms_cstime=0}) = 431499510
brk(0xd23b000) = 0xd23b000
munmap(0x2b3f70f01000, 4198400) = 0
munmap(0x2b3f70ede000, 143360) = 0
munmap(0x2b3f70ebb000, 143360) = 0
munmap(0x2b3f70e98000, 143360) = 0
exit_group(0)

However, I hope to know idea and some comment -)

Creating Database 11gR1 on ASM 11gR2

On Forums - show 10g database creating with 11gR2 ASM failed.
If we check on oracle support, we can use [ID 949073.1] ... That's helpful issue to fix.

To run older databases on 11gR2 ASM and Clusterware we need to create a persistent configuration for the nodes(pinning a node). Because Cluster configuration in 11gR2 is dynamic. This dynamic configuration is incompatible with older database releases.

On Oracle Document about "Pinning Cluster Nodes for Oracle Database Release 10.x or 11.x".

Time To Test It... (Creating Database 11gR1 on ASM 11gR2) and error.

ORA-19624: operation failed, retry possible
ORA-19870: error while restoring backup piece /oracle/product/11.1/db/assistants/dbca/templates/Seed_Database.dfb
ORA-19504: failed to create file "+DISK02"
ORA-17502: ksfdcre:4 Failed to create file +DISK02
ORA-15001: diskgroup "DISK02" does not exist or is not mounted
ORA-15077: could not locate ASM instance serving a required diskgroup
ORA-29701: unable to connect to Cluster Manager
ORA-06512: at "SYS.X$DBMS_BACKUP_RESTORE", line 5662
ORA-06512: at line 40

Check ASM... about database compatibility.

SQL> select name, compatibility, database_compatibility, state from v$asm_diskgroup_stat;

NAME COMPATIBILITY DATABASE_COMPAT STATE
------------------------------ --------------- --------------- -----------
DISK00 11.2.0.0.0 10.1.0.0.0 MOUNTED
DISK01 11.1.0.0.0 11.1.0.0.0 MOUNTED
DISK02 11.1.0.0.0 11.1.0.0.0 MOUNTED
DISK03 11.2.0.0.0 11.2.0.0.0 MOUNTED

then pin node, by crsctl pin css

# cd /oracle/grid2/
# cd bin/
# ./olsnodes -t -n
rhel5-test 1 Unpinned

# ./crsctl pin css -n rhel5-test
CRS-4664: Node rhel5-test successfully pinned.

# ./olsnodes -t -n
rhel5-test 1 Pinned

then create database again... and no problem.

then register my database(11gR1) in Grid.

$ srvctl add database -d orcl11gr1 -o /oracle/product/11.1/db -p +DISK02/orcl11gr1/spfileorcl11gr1.ora -y AUTOMATIC
$ srvctl add instance -d orcl11gr1 -i orcl11gr1 -n rhel5-test

$ srvctl start database -d orcl11gr1
$ srvctl status database -d orcl11gr1
Instance orcl11gr1 is running on node rhel5-test

$ ./crsstat ora.orcl11gr1
HA Resource Target State (Host)
----------- ------ -----
ora.orcl11gr1.db ONLINE ONLINE on rhel5-test

11gR2 Add VIP to Node

I'm just curious on Forums.

PRKO-2165 : VIP does not exist on node(s) : linux2

I'd like to know Add VIP to a node idea. So, Tested It.

Virtual IP = 192.168.254.92

/etc/hosts file:
192.168.254.92 RHEL5-TEST-VIP

# ./srvctl status nodeapps
Network is enabled
Network is running on node: rhel5-test
GSD is disabled
GSD is not running on node: rhel5-test
ONS is enabled
ONS daemon is running on node: rhel5-test
eONS is enabled
eONS daemon is running on node: rhel5-test
PRKO-2165 : VIP does not exist on node(s) : rhel5-test

# ./srvctl start nodeapps
PRKO-2421 : Network resource is already started on node(s): rhel5-test
PRKO-2439 : VIP does not exist.
PRKO-2422 : ONS is already started on node(s): rhel5-test
PRKO-2423 : eONS is already started on node(s): rhel5-test

$ ./crsstat vip
HA Resource Target State (Host)
----------- ------ -----
ora.scan1.vip ONLINE ONLINE on rhel5-test

# ping 192.168.254.92
PING 192.168.254.92 (192.168.254.92) 56(84) bytes of data.
From 192.168.254.91 icmp_seq=1 Destination Host Unreachable
From 192.168.254.91 icmp_seq=2 Destination Host Unreachable

--- 192.168.254.92 ping statistics ---
2 packets transmitted, 0 received, +3 errors, 100% packet loss, time 3999ms

Try to add VIP on my node.

srvctl add vip -n node_name -A {name|ip}/netmask[/if1[if2|...]]
   [-k network_number] [-v]

# ./srvctl add vip -n rhel5-test -A 192.168.254.92/255.255.255.0/eth0 -k 2
PRCN-2049 : The network attributes specified (network number: 2, subnet: 192.168.254.0, adapters: eth0) conflict with an already registered network (network number: 1, subnet: 192.168.254.0, adapters: eth0)

So, changed network_number:

# ./srvctl add vip -n rhel5-test -A 192.168.254.92/255.255.255.0/eth0 -k 1

Check again & again.

$ ./crsstat vip
HA Resource Target State (Host)
----------- ------ -----
ora.rhel5-test.vip OFFLINE OFFLINE
ora.scan1.vip ONLINE ONLINE on rhel5-test

# ping 192.168.254.92
PING 192.168.254.92 (192.168.254.92) 56(84) bytes of data.
From 192.168.254.91 icmp_seq=1 Destination Host Unreachable
From 192.168.254.91 icmp_seq=2 Destination Host Unreachable

--- 192.168.254.92 ping statistics ---
2 packets transmitted, 0 received, 100% packet loss, time 999ms

$ srvctl start nodeapps
PRKO-2421 : Network resource is already started on node(s): rhel5-test
PRKO-2422 : ONS is already started on node(s): rhel5-test
PRKO-2423 : eONS is already started on node(s): rhel5-test

Or

$ srvctl start vip -n rhel5-test

Check again.

$ ./crsstat vip
HA Resource Target State (Host)
----------- ------ -----
ora.rhel5-test.vip ONLINE ONLINE on rhel5-test
ora.scan1.vip ONLINE ONLINE on rhel5-test

# ping 192.168.254.92
PING 192.168.254.92 (192.168.254.92) 56(84) bytes of data.
64 bytes from 192.168.254.92: icmp_seq=1 ttl=64 time=0.043 ms
64 bytes from 192.168.254.92: icmp_seq=2 ttl=64 time=0.049 ms

--- 192.168.254.92 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 999ms

# ./srvctl config vip -n rhel5-test
VIP exists.:rhel5-test
VIP exists.: /192.168.254.92/192.168.254.92/255.255.255.0/eth0

That's a example to fun -)

By the way, we should add vip by name.
From /etc/hosts file:
192.168.254.92 RHEL5-TEST-VIP

# ./srvctl stop vip -n rhel5-test -f
# ./srvctl remove vip -i rhel5-test
Please confirm that you intend to remove the VIPs rhel5-test (y/[n]) y

Check:

$ ./crsstat vip
HA Resource Target State (Host)
----------- ------ -----
ora.scan1.vip ONLINE ONLINE on rhel5-test

start add VIP:

# ./srvctl add vip -n rhel5-test -A rhel5-test-vip/255.255.255.0/eth0 -k 1
# ./srvctl config vip -n rhel5-test
VIP exists.:rhel5-test
VIP exists.: /rhel5-test-vip/192.168.254.92/255.255.255.0/eth0

Check:

$ ./crsstat vip
HA Resource Target State (Host)
----------- ------ -----
ora.rhel5-test.vip OFFLINE OFFLINE
ora.scan1.vip ONLINE ONLINE on rhel5-test

then start...

# ./srvctl start vip -n rhel5-test
$ ./crsstat vip
HA Resource Target State (Host)
----------- ------ -----
ora.rhel5-test.vip ONLINE ONLINE on rhel5-test
ora.scan1.vip ONLINE ONLINE on rhel5-test

Nice -)

Oracle Data Guard 11g Handbook

I interested this book after I read chapter one and review book by Harald van Breederode.
I bought Oracle Data Guard 11g Handbook from Amazon and then read it. This's a great book. When we need to implement Data Guard. we must read.

Oracle Data Guard 11g Handbook book written by Larry Carpenter, Joseph Meeks, Charles Kim, Bill Burke, Sonya Carothers, Joydip Kundu, Michael Smith, Nitin Vengurlekar. This book has 14 chapters. They wrote about Data Guard Architecture/Implementing/Management/...

What will we learn from this book:

- Data Guard Architecture

- Data Guard and Data Guard Broker Implementing

- Rodo Processing

- Data Guard Monitoring

- Data Guard Troubleshooting

- Data Guard ...

Everything about Oracle Data Guard.

Reference:

Oracle Maximum Availability Architecture - MAA