Search In The Blog

Monday 24 July 2017

ORA-00600: internal error code, arguments: [kclgeneratecr_2]

ORA-00600: internal error code, arguments: [kclgeneratecr_2]


Symptoms:
This is two node RAC system (12.1.0.2) and One of the node rebooted with below error.
Errors in file /oracle/app/oracle12c/diag/rdbms/myrpt/MYRPT2/trace/MYRPT2_lms3_59442.trc (incident=320169):
ORA-00600: internal error code, arguments: [kclgeneratecr_2], [], [], [], [], [], [], [], [], [], [], []
Incident details in: /oracle/app/oracle12c/diag/rdbms/myrpt/MYRPT2/incident/incdir_320169/MYRPT2_lms3_59442_i320169.trc
Use ADRCI or Support Workbench to package the incident.

Solutions:
This is known bug, Patch 22454326 available for your version please download and apply

Patch 22454326 is one off patch and base PSU 12.1.0.2.170418 is required (similer to this Patch 22454326 available for the other base versions too).

LMS terminates the instance with ORA-00600: [kclgeneratecr_2] (Doc ID 2232271.1)

Steps Performed:

MYRPT1:/home/oracle>
MYRPT1:/home/oracle>srvctl status database -d MYRPT
Instance MYRPT1 is running on node myserver01
Instance MYRPT2 is running on node myserver02
MYRPT1:/home/oracle>
MYRPT1:/home/oracle>srvctl stop database -d MYRPT
MYRPT1:/home/oracle>
MYRPT1:/home/oracle>srvctl status database -d MYRPT
Instance MYRPT1 is not running on node myserver01
Instance MYRPT2 is not running on node myserver02
MYRPT1:/home/oracle>


MYRPT1:/home/oracle>
MYRPT1:/home/oracle>/bin/umount -t acfs /backup
umount: only root can do that
MYRPT1:/home/oracle>
MYRPT1:/home/oracle>logout
[root@myserver01 ngetadmin]#


[root@myserver01 ngetadmin]# df -kh
Filesystem                                       Size  Used Avail Use% Mounted on
/dev/mapper/vg_dbsrvr1-lv_root                    29G   17G   11G  62% /
tmpfs                                            127G  2.8G  124G   3% /dev/shm
/dev/mapper/vg_dbsrvr2-lv_agent                   20G  1.1G   18G   6% /agent
/dev/mapper/36005076043c14c381a9f09ba1ad22492p2  488M   94M  369M  21% /boot
/dev/mapper/36005076043c14c381a9f09ba1ad22492p1  200M  256K  200M   1% /boot/efi
/dev/mapper/vg_dbsrvr1-lv_dump                    29G   12G   17G  42% /dump
/dev/mapper/vg_dbsrvr2-lv_grid                    48G   38G  8.2G  83% /grid
/dev/mapper/vg_dbsrvr1-lv_home                    20G  8.0G   11G  44% /home
/dev/mapper/vg_dbsrvr2-lv_oracle                  96G   41G   51G  45% /oracle
/dev/mapper/vg_dbsrvr1-lv_tmp                     20G   13G  6.1G  67% /tmp
/dev/mapper/vg_dbsrvr1-lv_var                    9.5G  4.5G  4.6G  50% /var
/dev/asm/oggvol-358                               99G   30G   70G  30% /ogg
/dev/asm/backup-380                              2.0T  1.2T  837G  59% /backup
[root@myserver01 ngetadmin]#
[root@myserver01 ngetadmin]# /bin/umount -t acfs /backup
umount: /backup: device is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))
umount: /backup: device is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))
[root@myserver01 ngetadmin]#


stop os watcher

[root@myserver01 ngetadmin]# ps -ef | grep 134211
monitor  134211 115386  0 11:35 pts/1    00:00:00 /bin/bash
root     137648  55145  0 14:12 pts/4    00:00:00 grep 134211
[root@myserver01 ngetadmin]#
[root@myserver01 ngetadmin]# kill -9 134211
[root@myserver01 ngetadmin]#
[root@myserver01 ngetadmin]# lsof | grep /backup
bash      136145      root  cwd       DIR         252,194561      8192       2250 /backup/node1/oswbb
[root@myserver01 ngetadmin]# kill -9 136145
[root@myserver01 ngetadmin]#
[root@myserver01 ngetadmin]# lsof | grep /backup
[root@myserver01 ngetadmin]#
[root@myserver01 ngetadmin]#
[root@myserver01 ngetadmin]# /bin/umount -t acfs /backup
[root@myserver01 ngetadmin]#

[root@myserver01 ngetadmin]#
[root@myserver01 ngetadmin]# /bin/umount -t acfs /ogg
[root@myserver01 ngetadmin]#
[root@myserver01 ngetadmin]# df -kh
Filesystem                                       Size  Used Avail Use% Mounted on
/dev/mapper/vg_dbsrvr1-lv_root                    29G   17G   11G  62% /
tmpfs                                            127G  2.8G  124G   3% /dev/shm
/dev/mapper/vg_dbsrvr2-lv_agent                   20G  1.1G   18G   6% /agent
/dev/mapper/36005076043c14c381a9f09ba1ad22492p2  488M   94M  369M  21% /boot
/dev/mapper/36005076043c14c381a9f09ba1ad22492p1  200M  256K  200M   1% /boot/efi
/dev/mapper/vg_dbsrvr1-lv_dump                    29G   12G   17G  42% /dump
/dev/mapper/vg_dbsrvr2-lv_grid                    48G   38G  8.2G  83% /grid
/dev/mapper/vg_dbsrvr1-lv_home                    20G  8.0G   11G  44% /home
/dev/mapper/vg_dbsrvr2-lv_oracle                  96G   41G   51G  45% /oracle
/dev/mapper/vg_dbsrvr1-lv_tmp                     20G   13G  6.1G  67% /tmp
/dev/mapper/vg_dbsrvr1-lv_var                    9.5G  4.5G  4.6G  50% /var
[root@myserver01 ngetadmin]#

[root@myserver02 ngetadmin]#
[root@myserver02 ngetadmin]# /bin/umount -t acfs /backup
umount: /backup: device is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))
umount: /backup: device is busy.
        (In some cases useful info about processes that use
         the device is found by lsof(8) or fuser(1))
[root@myserver02 ngetadmin]#
[root@myserver02 ngetadmin]# lsof | grep /backup
bash       78137      root  cwd       DIR         252,194561      4096        722 /backup/node2/oswbb/archive
[root@myserver02 ngetadmin]#
[root@myserver02 ngetadmin]# kill -9 78137
[root@myserver02 ngetadmin]#
[root@myserver02 ngetadmin]#  /bin/umount -t acfs /backup
[root@myserver02 ngetadmin]#
[root@myserver02 ngetadmin]# /bin/umount -t acfs /ogg
[root@myserver02 ngetadmin]#
[root@myserver02 ngetadmin]# df -kh
Filesystem                                       Size  Used Avail Use% Mounted on
/dev/mapper/vg_dbsrvr1-lv_root                    29G   13G   15G  46% /
tmpfs                                            127G  639M  126G   1% /dev/shm
/dev/mapper/vg_dbsrvr2-lv_agent                   20G  1.1G   18G   6% /agent
/dev/mapper/36005076043c14f781a9f0df1505843c8p2  488M   94M  369M  21% /boot
/dev/mapper/36005076043c14f781a9f0df1505843c8p1  200M  256K  200M   1% /boot/efi
/dev/mapper/vg_dbsrvr1-lv_dump                    29G  4.0G   24G  15% /dump
/dev/mapper/vg_dbsrvr2-lv_grid                    48G   40G  6.0G  88% /grid
/dev/mapper/vg_dbsrvr1-lv_home                    20G  4.0G   15G  22% /home
/dev/mapper/vg_dbsrvr2-lv_oracle                  96G   35G   57G  38% /oracle
/dev/mapper/vg_dbsrvr1-lv_tmp                     20G   57M   19G   1% /tmp
/dev/mapper/vg_dbsrvr1-lv_var                    9.5G  3.2G  5.9G  35% /var
[root@myserver02 ngetadmin]#







[root@myserver01 ngetadmin]# ps -ef | grep d.bin
root       1416  55145  0 14:18 pts/4    00:00:00 grep d.bin
root      27051      1  0 Jun15 ?        06:30:08 /grid/12c/app/12.1.0.2/grid/bin/ohasd.bin reboot
root      27100      1  0 Jun15 ?        04:27:00 /grid/12c/app/12.1.0.2/grid/bin/orarootagent.bin
grid      27166      1  0 Jun15 ?        03:39:48 /grid/12c/app/12.1.0.2/grid/bin/oraagent.bin
grid      27179      1  0 Jun15 ?        01:31:09 /grid/12c/app/12.1.0.2/grid/bin/mdnsd.bin
grid      27181      1  0 Jun15 ?        05:26:56 /grid/12c/app/12.1.0.2/grid/bin/evmd.bin
grid      27197      1  0 Jun15 ?        01:41:36 /grid/12c/app/12.1.0.2/grid/bin/gpnpd.bin
grid      27209  27181  0 Jun15 ?        01:29:08 /grid/12c/app/12.1.0.2/grid/bin/evmlogger.bin -o /grid/12c/app/12.1.0.2/grid/log/[HOSTNAME]/evmd/evmlogger.info -l /grid/12c/app/12.1.0.2/grid/log/[HOSTNAME]/evmd/evmlogger.log
grid      27215      1  1 Jun15 ?        09:05:42 /grid/12c/app/12.1.0.2/grid/bin/gipcd.bin
root      27274      1  0 Jun15 ?        03:26:25 /grid/12c/app/12.1.0.2/grid/bin/cssdmonitor
root      27294      1  0 Jun15 ?        03:47:05 /grid/12c/app/12.1.0.2/grid/bin/cssdagent
grid      27307      1  5 Jun15 ?        1-09:20:40 /grid/12c/app/12.1.0.2/grid/bin/ocssd.bin
root      27454      1  0 Jun15 ?        05:31:30 /grid/12c/app/12.1.0.2/grid/bin/octssd.bin reboot
root      27672      1  5 Jun15 ?        1-10:05:43 /grid/12c/app/12.1.0.2/grid/bin/osysmond.bin
root      27687      1  1 Jun15 ?        10:13:34 /grid/12c/app/12.1.0.2/grid/bin/crsd.bin reboot
grid      27760      1  0 Jun15 ?        06:12:06 /grid/12c/app/12.1.0.2/grid/bin/oraagent.bin
root      27764      1  1 Jun15 ?        11:41:07 /grid/12c/app/12.1.0.2/grid/bin/orarootagent.bin
root      27942      1  2 Jun15 ?        19:08:54 /grid/12c/app/12.1.0.2/grid/bin/ologgerd -M -d /grid/12c/app/12.1.0.2/grid/crf/db/myserver01
grid      27974      1  0 Jun15 ?        01:47:48 /grid/12c/app/12.1.0.2/grid/bin/scriptagent.bin
grid      27978      1  0 Jun15 ?        00:02:03 /grid/12c/app/12.1.0.2/grid/bin/tnslsnr MGMTLSNR -no_crs_notify -inherit
oracle    28435      1  0 Jun15 ?        03:36:24 /grid/12c/app/12.1.0.2/grid/bin/oraagent.bin
grid      28441      1  0 Jun15 ?        00:02:23 /grid/12c/app/12.1.0.2/grid/bin/tnslsnr LISTENER -no_crs_notify -inherit
grid      28495      1  0 Jun15 ?        00:04:27 /grid/12c/app/12.1.0.2/grid/bin/tnslsnr LISTENER_SCAN2 -inherit
grid      28500      1  0 Jun15 ?        00:03:24 /grid/12c/app/12.1.0.2/grid/bin/tnslsnr LISTENER_SCAN3 -inherit
[root@myserver01 ngetadmin]#
[root@myserver01 ngetadmin]# cd /grid/12c/app/12.1.0.2/grid/bin/
[root@myserver01 bin]#
[root@myserver01 bin]# ./crsctl stop crs
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'myserver01'
CRS-2673: Attempting to stop 'ora.crsd' on 'myserver01'
CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on 'myserver01'
CRS-2673: Attempting to stop 'ora.CTRLDG2.dg' on 'myserver01'
CRS-2673: Attempting to stop 'ora.LISTENER_SCAN2.lsnr' on 'myserver01'
CRS-2673: Attempting to stop 'ora.LISTENER.lsnr' on 'myserver01'
CRS-2673: Attempting to stop 'ora.LISTENER_RDB.lsnr' on 'myserver01'
CRS-2673: Attempting to stop 'ora.LISTENER_SCAN3.lsnr' on 'myserver01'
CRS-2673: Attempting to stop 'ora.mgmtdb' on 'myserver01'
CRS-2673: Attempting to stop 'ora.OGGDG.OGGVOL.advm' on 'myserver01'
CRS-2673: Attempting to stop 'ora.oc4j' on 'myserver01'
CRS-2673: Attempting to stop 'ora.REDOC.dg' on 'myserver01'
CRS-2673: Attempting to stop 'ora.REDOB.dg' on 'myserver01'
CRS-2673: Attempting to stop 'ora.cvu' on 'myserver01'
CRS-2677: Stop of 'ora.OGGDG.OGGVOL.advm' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.cvu' on 'myserver01' succeeded
CRS-2672: Attempting to start 'ora.cvu' on 'myserver02'
CRS-2676: Start of 'ora.cvu' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.LISTENER_RDB.lsnr' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.LISTENER.lsnr' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.myserver01.vip' on 'myserver01'
CRS-2677: Stop of 'ora.REDOC.dg' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.LISTENER_SCAN2.lsnr' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.scan2.vip' on 'myserver01'
CRS-2677: Stop of 'ora.CTRLDG2.dg' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.LISTENER_SCAN3.lsnr' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.scan3.vip' on 'myserver01'
CRS-2677: Stop of 'ora.REDOB.dg' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.myserver01.vip' on 'myserver01' succeeded
CRS-2672: Attempting to start 'ora.myserver01.vip' on 'myserver02'
CRS-2677: Stop of 'ora.scan3.vip' on 'myserver01' succeeded
CRS-2672: Attempting to start 'ora.scan3.vip' on 'myserver02'
CRS-2677: Stop of 'ora.scan2.vip' on 'myserver01' succeeded
CRS-2672: Attempting to start 'ora.scan2.vip' on 'myserver02'
CRS-2677: Stop of 'ora.oc4j' on 'myserver01' succeeded
CRS-2672: Attempting to start 'ora.oc4j' on 'myserver02'
CRS-2677: Stop of 'ora.mgmtdb' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.MGMTLSNR' on 'myserver01'
CRS-2677: Stop of 'ora.MGMTLSNR' on 'myserver01' succeeded
CRS-2672: Attempting to start 'ora.MGMTLSNR' on 'myserver02'
CRS-2676: Start of 'ora.myserver01.vip' on 'myserver02' succeeded
CRS-2676: Start of 'ora.scan3.vip' on 'myserver02' succeeded
CRS-2672: Attempting to start 'ora.LISTENER_SCAN3.lsnr' on 'myserver02'
CRS-2676: Start of 'ora.scan2.vip' on 'myserver02' succeeded
CRS-2672: Attempting to start 'ora.LISTENER_SCAN2.lsnr' on 'myserver02'
CRS-2676: Start of 'ora.MGMTLSNR' on 'myserver02' succeeded
CRS-2672: Attempting to start 'ora.mgmtdb' on 'myserver02'
CRS-2676: Start of 'ora.LISTENER_SCAN3.lsnr' on 'myserver02' succeeded
CRS-2676: Start of 'ora.LISTENER_SCAN2.lsnr' on 'myserver02' succeeded
CRS-2676: Start of 'ora.oc4j' on 'myserver02' succeeded
CRS-2676: Start of 'ora.mgmtdb' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.ARCH.dg' on 'myserver01'
CRS-2673: Attempting to stop 'ora.OCRV.dg' on 'myserver01'
CRS-2673: Attempting to stop 'ora.BACKUP.BACKUP.advm' on 'myserver01'
CRS-2673: Attempting to stop 'ora.CTRLDG1.dg' on 'myserver01'
CRS-2673: Attempting to stop 'ora.CTRLDG3.dg' on 'myserver01'
CRS-2673: Attempting to stop 'ora.DATA.dg' on 'myserver01'
CRS-2673: Attempting to stop 'ora.DATA1.dg' on 'myserver01'
CRS-2673: Attempting to stop 'ora.DATA2.dg' on 'myserver01'
CRS-2673: Attempting to stop 'ora.OGGDG.dg' on 'myserver01'
CRS-2673: Attempting to stop 'ora.REDOA.dg' on 'myserver01'
CRS-2677: Stop of 'ora.BACKUP.BACKUP.advm' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.BACKUP.dg' on 'myserver01'
CRS-2677: Stop of 'ora.OCRV.dg' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.DATA2.dg' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.CTRLDG3.dg' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.DATA1.dg' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.CTRLDG1.dg' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.ARCH.dg' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.REDOA.dg' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.DATA.dg' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.OGGDG.dg' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.BACKUP.dg' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.asm' on 'myserver01'
CRS-2677: Stop of 'ora.asm' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.ons' on 'myserver01'
CRS-2677: Stop of 'ora.ons' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.net1.network' on 'myserver01'
CRS-2677: Stop of 'ora.net1.network' on 'myserver01' succeeded
CRS-2792: Shutdown of Cluster Ready Services-managed resources on 'myserver01' has completed
CRS-2677: Stop of 'ora.crsd' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.mdnsd' on 'myserver01'
CRS-2673: Attempting to stop 'ora.gpnpd' on 'myserver01'
CRS-2673: Attempting to stop 'ora.storage' on 'myserver01'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'myserver01'
CRS-2677: Stop of 'ora.storage' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.crf' on 'myserver01'
CRS-2673: Attempting to stop 'ora.ctssd' on 'myserver01'
CRS-2673: Attempting to stop 'ora.evmd' on 'myserver01'
CRS-2673: Attempting to stop 'ora.asm' on 'myserver01'
CRS-2677: Stop of 'ora.drivers.acfs' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.mdnsd' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.gpnpd' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.crf' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.ctssd' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.evmd' on 'myserver01' succeeded
CRS-2677: Stop of 'ora.asm' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'myserver01'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'myserver01'
CRS-2677: Stop of 'ora.cssd' on 'myserver01' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'myserver01'
CRS-2677: Stop of 'ora.gipcd' on 'myserver01' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'myserver01' has completed
CRS-4133: Oracle High Availability Services has been stopped.
[root@myserver01 bin]#


[root@myserver02 ngetadmin]# ps -ef | grep d.bin
grid      11800      1  0 14:18 ?        00:00:00 /grid/12c/app/12.1.0.2/grid/bin/scriptagent.bin
grid      11810  11800  0 14:18 ?        00:00:00 /bin/sh /grid/12c/app/12.1.0.2/grid/bin/oc4jctl start
grid      11813  11810  0 14:18 ?        00:00:00 /grid/12c/app/12.1.0.2/grid/perl/bin/perl -I/grid/12c/app/12.1.0.2/grid/lib /grid/12c/app/12.1.0.2/grid/bin/oc4jctl.pl start
grid      11840      1  0 14:18 ?        00:00:00 /grid/12c/app/12.1.0.2/grid/bin/tnslsnr MGMTLSNR -no_crs_notify -inherit
grid      12033      1  0 14:18 ?        00:00:00 /grid/12c/app/12.1.0.2/grid/bin/tnslsnr LISTENER_SCAN3 -inherit
grid      12053      1  1 14:18 ?        00:00:00 /grid/12c/app/12.1.0.2/grid/bin/tnslsnr LISTENER_SCAN2 -inherit
root      12075  50189  0 14:18 pts/1    00:00:00 grep d.bin
root      38902      1  0 May19 ?        00:00:00 /grid/12c/app/12.1.0.2/grid/bin/orarootagent.bin
root      55913      1  0 Jun15 ?        06:25:56 /grid/12c/app/12.1.0.2/grid/bin/ohasd.bin reboot
root      56002      1  0 Jun15 ?        03:45:08 /grid/12c/app/12.1.0.2/grid/bin/orarootagent.bin
grid      56074      1  0 Jun15 ?        03:46:00 /grid/12c/app/12.1.0.2/grid/bin/oraagent.bin
grid      56088      1  0 Jun15 ?        01:43:13 /grid/12c/app/12.1.0.2/grid/bin/mdnsd.bin
grid      56091      1  0 Jun15 ?        05:37:20 /grid/12c/app/12.1.0.2/grid/bin/evmd.bin
grid      56106      1  0 Jun15 ?        01:51:33 /grid/12c/app/12.1.0.2/grid/bin/gpnpd.bin
grid      56118  56091  0 Jun15 ?        01:42:13 /grid/12c/app/12.1.0.2/grid/bin/evmlogger.bin -o /grid/12c/app/12.1.0.2/grid/log/[HOSTNAME]/evmd/evmlogger.info -l /grid/12c/app/12.1.0.2/grid/log/[HOSTNAME]/evmd/evmlogger.log
grid      56124      1  0 Jun15 ?        06:34:13 /grid/12c/app/12.1.0.2/grid/bin/gipcd.bin
root      56153      1  0 Jun15 ?        02:09:16 /grid/12c/app/12.1.0.2/grid/bin/cssdmonitor
root      56171      1  0 Jun15 ?        02:12:23 /grid/12c/app/12.1.0.2/grid/bin/cssdagent
grid      56184      1  1 Jun15 ?        07:06:16 /grid/12c/app/12.1.0.2/grid/bin/ocssd.bin
root      56293      1  0 Jun15 ?        04:39:16 /grid/12c/app/12.1.0.2/grid/bin/octssd.bin reboot
root      56483      1  3 Jun15 ?        1-00:49:19 /grid/12c/app/12.1.0.2/grid/bin/osysmond.bin
root      56522      1  1 Jun15 ?        07:18:48 /grid/12c/app/12.1.0.2/grid/bin/crsd.bin reboot
root      56625      1  1 Jun15 ?        08:41:13 /grid/12c/app/12.1.0.2/grid/bin/orarootagent.bin
grid      56629      1  0 Jun15 ?        03:16:18 /grid/12c/app/12.1.0.2/grid/bin/oraagent.bin
oracle    56890      1  0 Jun15 ?        03:43:00 /grid/12c/app/12.1.0.2/grid/bin/oraagent.bin
grid      56904      1  0 Jun15 ?        00:01:37 /grid/12c/app/12.1.0.2/grid/bin/tnslsnr LISTENER -no_crs_notify -inherit
grid      56933      1  0 Jun15 ?        00:02:22 /grid/12c/app/12.1.0.2/grid/bin/tnslsnr LISTENER_SCAN1 -inherit
[root@myserver02 ngetadmin]#
[root@myserver02 ngetadmin]# cd /grid/12c/app/12.1.0.2/grid/bin/
[root@myserver02 bin]#
[root@myserver02 bin]#
[root@myserver02 bin]# ./crsctl stop crs
CRS-2791: Starting shutdown of Oracle High Availability Services-managed resources on 'myserver02'
CRS-2673: Attempting to stop 'ora.crsd' on 'myserver02'
CRS-2790: Starting shutdown of Cluster Ready Services-managed resources on 'myserver02'
CRS-2673: Attempting to stop 'ora.LISTENER_SCAN3.lsnr' on 'myserver02'
CRS-2673: Attempting to stop 'ora.LISTENER_SCAN2.lsnr' on 'myserver02'
CRS-2673: Attempting to stop 'ora.LISTENER_SCAN1.lsnr' on 'myserver02'
CRS-2673: Attempting to stop 'ora.DATA.dg' on 'myserver02'
CRS-2673: Attempting to stop 'ora.CTRLDG3.dg' on 'myserver02'
CRS-2673: Attempting to stop 'ora.oc4j' on 'myserver02'
CRS-2673: Attempting to stop 'ora.myserver02.vip' on 'myserver02'
CRS-2673: Attempting to stop 'ora.mgmtdb' on 'myserver02'
CRS-2673: Attempting to stop 'ora.cvu' on 'myserver02'
CRS-2673: Attempting to stop 'ora.BACKUP.BACKUP.advm' on 'myserver02'
CRS-2673: Attempting to stop 'ora.LISTENER_RDB.lsnr' on 'myserver02'
CRS-2673: Attempting to stop 'ora.LISTENER.lsnr' on 'myserver02'
CRS-2677: Stop of 'ora.cvu' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.BACKUP.BACKUP.advm' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.LISTENER_RDB.lsnr' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.LISTENER.lsnr' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.CTRLDG3.dg' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.LISTENER_SCAN2.lsnr' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.scan2.vip' on 'myserver02'
CRS-2677: Stop of 'ora.LISTENER_SCAN3.lsnr' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.scan3.vip' on 'myserver02'
CRS-2677: Stop of 'ora.LISTENER_SCAN1.lsnr' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.scan1.vip' on 'myserver02'
CRS-2677: Stop of 'ora.DATA.dg' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.myserver02.vip' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.myserver01.vip' on 'myserver02'
CRS-2677: Stop of 'ora.scan2.vip' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.mgmtdb' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.ARCH.dg' on 'myserver02'
CRS-2673: Attempting to stop 'ora.BACKUP.dg' on 'myserver02'
CRS-2673: Attempting to stop 'ora.CTRLDG1.dg' on 'myserver02'
CRS-2673: Attempting to stop 'ora.CTRLDG2.dg' on 'myserver02'
CRS-2673: Attempting to stop 'ora.DATA1.dg' on 'myserver02'
CRS-2673: Attempting to stop 'ora.DATA2.dg' on 'myserver02'
CRS-2673: Attempting to stop 'ora.OCRV.dg' on 'myserver02'
CRS-2673: Attempting to stop 'ora.OGGDG.OGGVOL.advm' on 'myserver02'
CRS-2673: Attempting to stop 'ora.REDOA.dg' on 'myserver02'
CRS-2673: Attempting to stop 'ora.REDOB.dg' on 'myserver02'
CRS-2673: Attempting to stop 'ora.REDOC.dg' on 'myserver02'
CRS-2673: Attempting to stop 'ora.MGMTLSNR' on 'myserver02'
CRS-2677: Stop of 'ora.OGGDG.OGGVOL.advm' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.OGGDG.dg' on 'myserver02'
CRS-2677: Stop of 'ora.OCRV.dg' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.scan3.vip' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.MGMTLSNR' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.oc4j' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.CTRLDG2.dg' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.scan1.vip' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.CTRLDG1.dg' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.myserver01.vip' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.ARCH.dg' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.REDOC.dg' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.REDOB.dg' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.DATA2.dg' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.DATA1.dg' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.REDOA.dg' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.BACKUP.dg' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.OGGDG.dg' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.asm' on 'myserver02'
CRS-2677: Stop of 'ora.asm' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.ons' on 'myserver02'
CRS-2677: Stop of 'ora.ons' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.net1.network' on 'myserver02'
CRS-2677: Stop of 'ora.net1.network' on 'myserver02' succeeded
CRS-2792: Shutdown of Cluster Ready Services-managed resources on 'myserver02' has completed
CRS-2677: Stop of 'ora.crsd' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.storage' on 'myserver02'
CRS-2673: Attempting to stop 'ora.drivers.acfs' on 'myserver02'
CRS-2673: Attempting to stop 'ora.mdnsd' on 'myserver02'
CRS-2673: Attempting to stop 'ora.gpnpd' on 'myserver02'
CRS-2677: Stop of 'ora.storage' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.asm' on 'myserver02'
CRS-2677: Stop of 'ora.drivers.acfs' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.mdnsd' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.gpnpd' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.asm' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.cluster_interconnect.haip' on 'myserver02'
CRS-2677: Stop of 'ora.cluster_interconnect.haip' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.ctssd' on 'myserver02'
CRS-2673: Attempting to stop 'ora.evmd' on 'myserver02'
CRS-2677: Stop of 'ora.evmd' on 'myserver02' succeeded
CRS-2677: Stop of 'ora.ctssd' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.cssd' on 'myserver02'
CRS-2677: Stop of 'ora.cssd' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.crf' on 'myserver02'
CRS-2677: Stop of 'ora.crf' on 'myserver02' succeeded
CRS-2673: Attempting to stop 'ora.gipcd' on 'myserver02'
CRS-2677: Stop of 'ora.gipcd' on 'myserver02' succeeded
CRS-2793: Shutdown of Oracle High Availability Services-managed resources on 'myserver02' has completed
CRS-4133: Oracle High Availability Services has been stopped.
[root@myserver02 bin]#
[root@myserver02 bin]#

MYRPT1:/grid/12c_grid_software/22454326>which opatch
/oracle/app/oracle12c/product/12.1.0.2/rdb/OPatch/opatch
MYRPT1:/grid/12c_grid_software/22454326>
MYRPT1:/grid/12c_grid_software/22454326>id
uid=551(oracle) gid=550(oinstall) groups=550(oinstall),551(dba),553(asmdba)
MYRPT1:/grid/12c_grid_software/22454326>
MYRPT1:/grid/12c_grid_software/22454326>opatch apply
Oracle Interim Patch Installer version 12.2.0.1.8
Copyright (c) 2017, Oracle Corporation.  All rights reserved.


Oracle Home       : /oracle/app/oracle12c/product/12.1.0.2/rdb
Central Inventory : /grid/oraInventory
   from           : /oracle/app/oracle12c/product/12.1.0.2/rdb/oraInst.loc
OPatch version    : 12.2.0.1.8
OUI version       : 12.1.0.2.0
Log file location : /oracle/app/oracle12c/product/12.1.0.2/rdb/cfgtoollogs/opatch/opatch2017-07-12_14-21-55PM_1.log

Verifying environment and performing prerequisite checks...
OPatch continues with these patches:   22454326

Do you want to proceed? [y|n]
y
User Responded with: Y
All checks passed.

This node is part of an Oracle Real Application Cluster.
Remote nodes: 'myserver02'
Local node: 'myserver01'
Please shutdown Oracle instances running out of this ORACLE_HOME on the local system.
(Oracle Home = '/oracle/app/oracle12c/product/12.1.0.2/rdb')


Is the local system ready for patching? [y|n]
y
User Responded with: Y
Backing up files...
Applying interim patch '22454326' to OH '/oracle/app/oracle12c/product/12.1.0.2/rdb'

Patching component oracle.rdbms, 12.1.0.2.0...

Patching in rolling mode.


The node 'myserver02' will be patched next.


Please shutdown Oracle instances running out of this ORACLE_HOME on 'myserver02'.
(Oracle Home = '/oracle/app/oracle12c/product/12.1.0.2/rdb')

Is the node ready for patching? [y|n]
y
User Responded with: Y
Updating nodes 'myserver02'
   Apply-related files are:
     FP = "/oracle/app/oracle12c/product/12.1.0.2/rdb/.patch_storage/NApply/2017-07-12_14-21-55PM/rac/copy_files.txt"
     DP = "/oracle/app/oracle12c/product/12.1.0.2/rdb/.patch_storage/NApply/2017-07-12_14-21-55PM/rac/copy_dirs.txt"
     MP = "/oracle/app/oracle12c/product/12.1.0.2/rdb/.patch_storage/NApply/2017-07-12_14-21-55PM/rac/make_cmds.txt"
     RC = "/oracle/app/oracle12c/product/12.1.0.2/rdb/.patch_storage/NApply/2017-07-12_14-21-55PM/rac/remote_cmds.txt"

Instantiating the file "/oracle/app/oracle12c/product/12.1.0.2/rdb/.patch_storage/NApply/2017-07-12_14-21-55PM/rac/copy_files.txt.instantiated" by replacing $ORACLE_HOME in "/oracle/app/oracle12c/product/12.1.0.2/rdb/.patch_storage/NApply/2017-07-12_14-21-55PM/rac/copy_files.txt" with actual path.
Propagating files to remote nodes...
Instantiating the file "/oracle/app/oracle12c/product/12.1.0.2/rdb/.patch_storage/NApply/2017-07-12_14-21-55PM/rac/copy_dirs.txt.instantiated" by replacing $ORACLE_HOME in "/oracle/app/oracle12c/product/12.1.0.2/rdb/.patch_storage/NApply/2017-07-12_14-21-55PM/rac/copy_dirs.txt" with actual path.
Propagating directories to remote nodes...
Instantiating the file "/oracle/app/oracle12c/product/12.1.0.2/rdb/.patch_storage/NApply/2017-07-12_14-21-55PM/rac/make_cmds.txt.instantiated" by replacing $ORACLE_HOME in "/oracle/app/oracle12c/product/12.1.0.2/rdb/.patch_storage/NApply/2017-07-12_14-21-55PM/rac/make_cmds.txt" with actual path.
Running command on remote node 'myserver02':
cd /oracle/app/oracle12c/product/12.1.0.2/rdb/rdbms/lib; /usr/bin/make -f ins_rdbms.mk ioracle ORACLE_HOME=/oracle/app/oracle12c/product/12.1.0.2/rdb || echo REMOTE_MAKE_FAILED::>&2


The node 'myserver02' has been patched.  You can restart Oracle instances on it.


Finished patching in rolling mode.

Patch 22454326 successfully applied.
Log file location: /oracle/app/oracle12c/product/12.1.0.2/rdb/cfgtoollogs/opatch/opatch2017-07-12_14-21-55PM_1.log

OPatch succeeded.
MYRPT1:/grid/12c_grid_software/22454326>



MYRPT1:/grid/12c_grid_software/22454326>opatch lsinventory
Oracle Interim Patch Installer version 12.2.0.1.8
Copyright (c) 2017, Oracle Corporation.  All rights reserved.


Oracle Home       : /oracle/app/oracle12c/product/12.1.0.2/rdb
Central Inventory : /grid/oraInventory
   from           : /oracle/app/oracle12c/product/12.1.0.2/rdb/oraInst.loc
OPatch version    : 12.2.0.1.8
OUI version       : 12.1.0.2.0
Log file location : /oracle/app/oracle12c/product/12.1.0.2/rdb/cfgtoollogs/opatch/opatch2017-07-12_14-23-44PM_1.log

Lsinventory Output file location : /oracle/app/oracle12c/product/12.1.0.2/rdb/cfgtoollogs/opatch/lsinv/lsinventory2017-07-12_14-23-44PM.txt

--------------------------------------------------------------------------------
Local Machine Information::
Hostname: myserver01.cris.org.in
ARU platform id: 226
ARU platform description:: Linux x86-64

Installed Top-level Products (1):

Oracle Database 12c                                                  12.1.0.2.0
There are 1 products installed in this Oracle Home.


Interim patches (3) :

Patch  22454326     : applied on Wed Jul 12 14:22:21 IST 2017
Unique Patch ID:  21030602
   Created on 18 Feb 2017, 04:30:11 hrs PST8PDT
   Bugs fixed:
     22454326
   This patch overlays patches:
     25171037
   This patch needs patches:
     25171037
   as prerequisites

Patch  25363740     : applied on Sat May 20 00:42:31 IST 2017
Unique Patch ID:  20990699
Patch description:  "OCW Patch Set Update : 12.1.0.2.170418 (25363740)"
   Created on 7 Mar 2017, 23:34:43 hrs PST8PDT
   Bugs fixed:
     21339083, 18589889, 20768643, 19061429, 19133945, 19341538, 19521169
     20011424, 19361757, 19187207, 19302350, 19130141, 19530755, 21441387
     19028836, 19699720, 19620918, 19168690, 18899171, 17005215, 19244316
     19653795, 18634372, 19471722, 19536337, 19184188, 19131709, 20160490
     20995001, 21131037, 19380733, 20985167, 18943696, 20552947, 19591984
     18135723, 18461246, 20014326, 19524857, 25250824, 18890943, 19154753
     19499021, 25206628, 19046190, 20923930, 20028666, 24451580, 19051385
     19050688, 19682695, 19831611, 22891868, 21607972, 18922918, 21087799
     18956780, 19683886, 23757020, 19026993, 20110156, 21269977, 18261648
     18362971, 19558324, 22651583, 20185476, 18882642, 19478119, 19414274
     17447588, 19262534, 20752167, 19147513, 22475834, 18893463, 21242118
     19514085, 20758997, 19659885, 19319904, 18703978, 23079844, 24796183
     19700256, 19703246, 20467806, 19192901, 21527118, 19479503, 19147751
     18901356, 21232394, 18964974, 22162062, 18835366, 19013789, 19207286
     18950232, 22583325, 19148791, 19556820, 16505171, 21228405, 22261732
     22135419, 19449737, 21878613, 21646169, 19187515, 19230771, 20795241
     19453778, 19853036, 18843572, 18520351, 21430567, 19185148, 19232454
     21220597, 18541110, 18834955, 19204743, 20175174, 19140891, 18747335
     20676340, 19270660, 19325701, 20877003, 19222693, 19662663, 19684131
     19010177, 19272663, 18910576, 18907170, 19426384, 9372680, 20737462
     19700294, 20289828, 19331454, 22568500, 20278216, 20827053, 20119857
     19703199, 22456393, 20588164, 18955644, 20991879, 18508710, 19908346
     19589221, 18952857, 19537762, 21152052, 16286734, 20669114, 18762843
     18945249, 19184799, 19205086, 19351363, 20091753, 19031737, 19079087
     19148367, 18968981, 19150517, 20231741, 19217019, 18730096, 18975620
     19067804, 21259359, 19054979, 19209951, 19318983, 18747382, 19154673
     18752378, 22905714, 19150088, 19013444, 19234177, 18999857, 19075747
     19367276, 19612597, 19874047, 19677813, 23049292, 19427050, 18910443
     20033787, 19148982, 19777496, 20730170, 24362631, 18523468, 19277814
     20562133, 19071526, 21420753, 25537905, 18850051, 20433932, 20061168
     18468128, 19370739, 20822591, 19609388, 18776786, 20136892, 18989446
     19148793, 22904978, 19585454, 18317489, 19955755, 19059053, 19807548
     19124972, 22229116, 18910748, 19147509, 18849896, 19617592, 19398098
     22840085, 19076165, 19498411, 19704993, 21294273, 19459023, 20634511
     21229744, 19516572, 21474380, 21484367, 20276459, 21550724, 17208793
     19234907, 20007009, 19538714, 22867050, 22728264, 19383028, 19513969
     22195657, 23111368, 19986391, 19314048, 18834934, 19473851, 19458082
     18894342, 19522067, 23618734, 18116578, 19241857, 19076778, 19066699
     25206563, 19068003, 18937186, 20267877, 19916737, 19770543, 22863466
     19635215, 19613533, 21057239, 20783416, 19315522, 19141785, 19163887
     19885321, 25233268, 18715868, 18852058, 19701617, 19280860, 19139608
     21453435, 18946768, 19135521, 18604143, 23150001, 19627476, 20568884
     20317221, 20177779, 19266658, 19491996, 24522717, 18330979, 19027351
     21227730, 18707416, 21118157, 21113068, 19925992, 20235486, 20006646
     15922206, 18439295, 18991776, 21245437, 23170839, 19163425, 19550195
     20389257, 18849021, 18861196, 17940721, 18524834, 19287706, 19522313
     18835283, 18748932, 19184765, 21639478, 14851828, 20598625, 19226141
     19053891, 18871287, 25371730, 18998228, 18980002, 19682778, 19325604
     8934841, 18777835, 17338864, 18945906, 19513650, 19702758, 18952577
     19473088, 19178517, 18729269, 19529729, 19455563, 20877072, 23275930
     20340620, 18536826, 19292605, 22264951, 20920711, 20660273, 20011635
     17592037, 18935857, 22989069, 19029647, 19179158, 22302421, 23076471
     19140712, 20012766, 23049301, 19184276, 25206643, 20510208, 20001507
     20079414, 19680763, 19259765, 18962892, 19513888, 14661880, 21519340
     21222147, 20218012, 19551830, 21616278, 19068333, 18945435, 20775015
     19319192, 20839581, 19178629, 18774591, 21694237, 19304104, 19284357
     22594310, 20725084, 19457575, 21297657, 19069755, 19021575, 18715884
     19584688, 18798573, 19812592, 19018001, 19292272, 19270956, 21967791
     21794779, 23055224, 18700893, 21276514, 20467377, 18406774, 20607821
     21976167, 21082737, 21694632, 20115586, 20883009, 19164099, 18283792
     19155350, 19884033, 19146822, 18798432, 21123424, 24356071, 19471836
     20936562, 18750781, 19045143, 20559126, 22483804, 19146980, 19601003
     20099624, 21378016, 19537547, 19281106, 18862203, 19954372, 18235274
     19205617, 19513351, 20219458, 19150313, 18843054, 18708349, 19332386
     18953639, 21423055, 19371270, 25221056, 19203996, 20038431, 19370846
     21053036, 18998379, 20157569, 21357522, 19273577, 24709351, 25206634
     22113170, 19632437, 18990354, 19288396, 20620581, 19557558, 19127078
     22548137, 21792881, 20900588, 20053557, 22088198, 22664609, 19315567
     19444471, 19154933, 20953798, 18290252, 18813323, 19915945, 19500293
     18643483, 19134098, 23034352, 25206614, 18965694, 19226858, 19591356
     19602208, 22884308, 20069197, 18417590, 18920408, 18263259, 18636884
     22289350, 19043795, 20746507, 18260170, 18919682, 18678829, 19319357
     19273758, 18953878, 18999195, 24314323, 18759724, 20458032, 19066844
     13843841, 20290467, 22934855, 19649640, 19062675, 21924910, 18859710
     20862812, 19504641, 23101611, 19341481, 20293730, 19343245, 18304090
     19241655, 19470791, 18242738, 20649573, 18372060, 18953889, 18827679
     19140711, 19023430, 19259290, 19045388, 20186278, 25197395, 19522571
     19709663, 18875012, 18861564, 21075392, 19273760, 19225265, 15885689
     20853464, 21696068, 18819158, 21394146, 19049721, 19368917, 25206650
     21255373, 22393909, 20408163, 20579351, 18868829, 20350915, 21327402
     22024217, 23101338, 19820247, 25206585, 20363708, 19538241, 19804032

Patch  25171037     : applied on Sat May 20 00:41:36 IST 2017
Unique Patch ID:  21099266
Patch description:  "Database Patch Set Update : 12.1.0.2.170418 (25171037)"
   Created on 7 Mar 2017, 12:37:23 hrs PST8PDT
Sub-patch  24732082; "Database Patch Set Update : 12.1.0.2.170117 (24732082)"
Sub-patch  24006101; "Database Patch Set Update : 12.1.0.2.161018 (24006101)"
Sub-patch  23054246; "Database Patch Set Update : 12.1.0.2.160719 (23054246)"
Sub-patch  22291127; "Database Patch Set Update : 12.1.0.2.160419 (22291127)"
Sub-patch  21948354; "Database Patch Set Update : 12.1.0.2.160119 (21948354)"
Sub-patch  21359755; "Database Patch Set Update : 12.1.0.2.5 (21359755)"
Sub-patch  20831110; "Database Patch Set Update : 12.1.0.2.4 (20831110)"
Sub-patch  20299023; "Database Patch Set Update : 12.1.0.2.3 (20299023)"
Sub-patch  19769480; "Database Patch Set Update : 12.1.0.2.2 (19769480)"
   Bugs fixed:
     21099555, 22175564, 19141838, 22083366, 20842388, 20117253, 19865345
     19791273, 21542577, 20951038, 19243521, 22165897, 17008068, 19908836
     21281532, 19238590, 24577566, 21184223, 19134173, 20569094, 20031873
     20322560, 20387265, 21575362, 19149990, 21263635, 17551063, 18886413
     22160989, 22507210, 19366375, 19703301, 19001390, 24285405, 18202441
     20267166, 19358317, 19706965, 19068970, 18549238, 24739928, 18797519
     22148226, 20825533, 21196809, 19649152, 19670108, 18940497, 18948177
     22496904, 18964978, 19176326, 19035573, 20413820, 19176223, 21106027
     20904530, 20134339, 19074147, 20868862, 18411216, 25475853, 21322887
     21072646, 22507234, 20425790, 20862087, 18966843, 21329301, 20562898
     19333670, 20124446, 19468991, 19883092, 20878790, 18510194, 19658708
     19591608, 19402853, 20618595, 21787056, 22380919, 19469538, 21266085
     17835294, 19721304, 19068610, 19791377, 22178855, 16777441, 22173980
     20048359, 20746251, 21896069, 19185876, 20898391, 20907061, 20281121
     6599380, 19577410, 22092979, 19001359, 20603378, 23089357, 21387964
     19490948, 22294260, 17532734, 20832516, 22351572, 19309466, 20627866
     19081128, 20844426, 21188532, 18791688, 20890311, 21442094, 20596234
     20368850, 18973548, 19303936, 21296029, 20882568, 19461270, 21479753
     22077517, 20936905, 20235511, 21220620, 18964939, 19430401, 22296366
     21153266, 19409212, 20703000, 22657942, 19879746, 20657441, 21294938
     19684504, 19024808, 20528052, 24693382, 20977794, 18799993, 20466322
     18740837, 19662635, 18440095, 20228093, 19065556, 20212067, 21868720
     22905130, 19524384, 24350831, 17722075, 20446883, 25056052, 18952989
     24523374, 16870214, 19928926, 19835133, 21629064, 21354456, 20466628
     24386767, 25490238, 19931709, 19730508, 18819908, 20250147, 23124895
     23220453, 19188927, 20074391, 18307021, 20356733, 14643995, 19065677
     19547370, 21960504, 21225209, 20397490, 18967382, 19174430, 21241829
     19536415, 19171086, 21889720, 22465352, 22168163, 19335438, 24397438
     20447445, 18856999, 19869255, 20471920, 21620471, 23096938, 18990693
     19124336, 17890099, 24812585, 18990023, 21300341, 20101006, 20848335
     21744290, 20897759, 21668627, 19304354, 20543011, 19052488, 20794034
     23025340, 23260854, 18681056, 20952966, 19896336, 25539063, 18618122
     20328248, 20440930, 18456643, 19699191, 19201867, 22865673, 22022760
     20798891, 18743542, 25161298, 20347562, 22551446, 19777862, 19687159
     21373076, 19174942, 20424899, 21899588, 18899974, 21476308, 20598042
     21297872, 24308635, 20171986, 19058490, 19032777, 22815955, 19399918
     19434529, 21273804, 19018447, 22757364, 18851894, 19022470, 19284031
     18043064, 20173897, 22062026, 20475845, 17274537, 19440586, 18974476
     24825843, 22374754, 16887946, 17319928, 20401975, 20708701, 22062517
     22809871, 17655240, 16439813, 19805359, 19155797, 20859910, 19393542
     22024071, 17210525, 19189525, 21847223, 21649497, 25079710, 19075256
     20315311, 22762046, 22075064, 20936731, 18845653, 19280225, 19248799
     20560611, 18988834, 21756699, 18921743, 20245930, 18799063, 20373598
     19571367, 20476175, 20925795, 19018206, 25264559, 20711718, 20509482
     20181030, 20588502, 21911701, 18849537, 23501901, 19183343, 21917884
     21142837, 19189317, 19644859, 19390567, 19279273, 20669434, 16863642
     22528741, 25546608, 19619732, 18607546, 20348653, 19315691, 19676905
     20165574, 17867700, 20558005, 20734332, 19532017, 20922010, 19818513
     19450314, 22353346, 16941434, 20361671, 20009833, 22366558, 20294666
     18191823, 23197103, 19195895, 19371175, 19307662, 19154375, 20043616
     21977392, 18914624, 22529728, 25330273, 20139391, 19593445, 21291274
     19382851, 19520602, 19174521, 21875360, 19676012, 19326908, 20217801
     20093776, 18840932, 21097043, 21246723, 20803014, 21665897, 19143550
     20428621, 19627012, 14283239, 21422580, 19213447, 19518079, 18610915
     18674024, 24413809, 18306996, 19915271, 19524158, 20122715, 20284155
     20017509, 22359063, 19363645, 19597439, 21239530, 19383839, 20880215
     21756677, 19888853, 22458049, 19534363, 19354335, 19044962, 19639483
     19475971, 22353199, 22243719, 21060755, 22916353, 20378086, 24808595
     21756661, 21260431, 22923409, 19028800, 20877664, 21059919, 20879889
     21380789, 19723336, 19077215, 19604659, 21421886, 21285458, 23533524
     23170620, 22365117, 18288842, 19048007, 19308965, 19689979, 19503821
     21526048, 19197175, 19180770, 19902195, 23324000, 20318889, 19013183
     20591183, 19012119, 20464614, 19067244, 21632821, 19841800, 19512341
     22695831, 20331945, 19587324, 24316947, 19578350, 19637186, 19054077
     18674047, 19708632, 20898997, 21091431, 19289642, 21133343, 20869721
     21172913, 19258504, 17365043, 21419850, 19468347, 21373473, 25093739
     16359751, 21164318, 22520320, 19769480, 19439759, 19272708, 19978542
     19329654, 20402832, 19873610, 23229229, 13542050, 21517440, 19291380
     21915719, 20879709, 20677396, 19076343, 19561643, 19990037, 19487147
     18909599, 20831538, 19016730, 18250893, 16619249, 18354830, 24411921
     16756406, 18254023, 21188584, 19989009, 17414008, 20688221, 20704450
     20441797, 25483815, 19157754, 24437510, 18885870, 21785691, 20673810
     21450666, 18893947, 18705806, 22223463, 16923858, 18417036, 23314180
     20919320, 20474192, 22046677, 21299490, 19501299, 19385656, 20920911
     20899461, 21387128, 21315084, 18122373, 20581111, 19606174, 24690216
     18436647, 19023822, 19124589, 19178851, 19597583, 18499088, 19050649



Rac system comprising of multiple nodes
  Local node = myserver01
  Remote node = myserver02

--------------------------------------------------------------------------------
OPatch succeeded.
MYRPT1:/grid/12c_grid_software/22454326>
MYRPT1:/grid/12c_grid_software/22454326>opatch lsinventory | grep 22454326
Patch  22454326     : applied on Wed Jul 12 14:22:21 IST 2017
     22454326
MYRPT1:/grid/12c_grid_software/22454326>


MYRPT2:/home/oracle>which opatch
/oracle/app/oracle12c/product/12.1.0.2/rdb/OPatch/opatch
MYRPT2:/home/oracle>
MYRPT2:/home/oracle>opatch lsinventory | grep 22454326
Patch  22454326     : applied on Wed Jul 12 14:22:21 IST 2017
     22454326
MYRPT2:/home/oracle>
MYRPT2:/home/oracle>opatch lsinventory
Oracle Interim Patch Installer version 12.2.0.1.8
Copyright (c) 2017, Oracle Corporation.  All rights reserved.


Oracle Home       : /oracle/app/oracle12c/product/12.1.0.2/rdb
Central Inventory : /grid/oraInventory
   from           : /oracle/app/oracle12c/product/12.1.0.2/rdb/oraInst.loc
OPatch version    : 12.2.0.1.8
OUI version       : 12.1.0.2.0
Log file location : /oracle/app/oracle12c/product/12.1.0.2/rdb/cfgtoollogs/opatch/opatch2017-07-12_14-25-42PM_1.log

Lsinventory Output file location : /oracle/app/oracle12c/product/12.1.0.2/rdb/cfgtoollogs/opatch/lsinv/lsinventory2017-07-12_14-25-42PM.txt

--------------------------------------------------------------------------------
Local Machine Information::
Hostname: myserver02.cris.org.in
ARU platform id: 226
ARU platform description:: Linux x86-64

Installed Top-level Products (1):

Oracle Database 12c                                                  12.1.0.2.0
There are 1 products installed in this Oracle Home.


Interim patches (3) :

Patch  22454326     : applied on Wed Jul 12 14:22:21 IST 2017
Unique Patch ID:  21030602
   Created on 18 Feb 2017, 04:30:11 hrs PST8PDT
   Bugs fixed:
     22454326
   This patch overlays patches:
     25171037
   This patch needs patches:
     25171037
   as prerequisites

Patch  25363740     : applied on Sat May 20 00:42:31 IST 2017
Unique Patch ID:  20990699
Patch description:  "OCW Patch Set Update : 12.1.0.2.170418 (25363740)"
   Created on 7 Mar 2017, 23:34:43 hrs PST8PDT
   Bugs fixed:
     21339083, 18589889, 20768643, 19061429, 19133945, 19341538, 19521169
     20011424, 19361757, 19187207, 19302350, 19130141, 19530755, 21441387
     19028836, 19699720, 19620918, 19168690, 18899171, 17005215, 19244316
     19653795, 18634372, 19471722, 19536337, 19184188, 19131709, 20160490
     20995001, 21131037, 19380733, 20985167, 18943696, 20552947, 19591984
     18135723, 18461246, 20014326, 19524857, 25250824, 18890943, 19154753
     19499021, 25206628, 19046190, 20923930, 20028666, 24451580, 19051385
     19050688, 19682695, 19831611, 22891868, 21607972, 18922918, 21087799
     18956780, 19683886, 23757020, 19026993, 20110156, 21269977, 18261648
     18362971, 19558324, 22651583, 20185476, 18882642, 19478119, 19414274
     17447588, 19262534, 20752167, 19147513, 22475834, 18893463, 21242118
     19514085, 20758997, 19659885, 19319904, 18703978, 23079844, 24796183
     19700256, 19703246, 20467806, 19192901, 21527118, 19479503, 19147751
     18901356, 21232394, 18964974, 22162062, 18835366, 19013789, 19207286
     18950232, 22583325, 19148791, 19556820, 16505171, 21228405, 22261732
     22135419, 19449737, 21878613, 21646169, 19187515, 19230771, 20795241
     19453778, 19853036, 18843572, 18520351, 21430567, 19185148, 19232454
     21220597, 18541110, 18834955, 19204743, 20175174, 19140891, 18747335
     20676340, 19270660, 19325701, 20877003, 19222693, 19662663, 19684131
     19010177, 19272663, 18910576, 18907170, 19426384, 9372680, 20737462
     19700294, 20289828, 19331454, 22568500, 20278216, 20827053, 20119857
     19703199, 22456393, 20588164, 18955644, 20991879, 18508710, 19908346
     19589221, 18952857, 19537762, 21152052, 16286734, 20669114, 18762843
     18945249, 19184799, 19205086, 19351363, 20091753, 19031737, 19079087
     19148367, 18968981, 19150517, 20231741, 19217019, 18730096, 18975620
     19067804, 21259359, 19054979, 19209951, 19318983, 18747382, 19154673
     18752378, 22905714, 19150088, 19013444, 19234177, 18999857, 19075747
     19367276, 19612597, 19874047, 19677813, 23049292, 19427050, 18910443
     20033787, 19148982, 19777496, 20730170, 24362631, 18523468, 19277814
     20562133, 19071526, 21420753, 25537905, 18850051, 20433932, 20061168
     18468128, 19370739, 20822591, 19609388, 18776786, 20136892, 18989446
     19148793, 22904978, 19585454, 18317489, 19955755, 19059053, 19807548
     19124972, 22229116, 18910748, 19147509, 18849896, 19617592, 19398098
     22840085, 19076165, 19498411, 19704993, 21294273, 19459023, 20634511
     21229744, 19516572, 21474380, 21484367, 20276459, 21550724, 17208793
     19234907, 20007009, 19538714, 22867050, 22728264, 19383028, 19513969
     22195657, 23111368, 19986391, 19314048, 18834934, 19473851, 19458082
     18894342, 19522067, 23618734, 18116578, 19241857, 19076778, 19066699
     25206563, 19068003, 18937186, 20267877, 19916737, 19770543, 22863466
     19635215, 19613533, 21057239, 20783416, 19315522, 19141785, 19163887
     19885321, 25233268, 18715868, 18852058, 19701617, 19280860, 19139608
     21453435, 18946768, 19135521, 18604143, 23150001, 19627476, 20568884
     20317221, 20177779, 19266658, 19491996, 24522717, 18330979, 19027351
     21227730, 18707416, 21118157, 21113068, 19925992, 20235486, 20006646
     15922206, 18439295, 18991776, 21245437, 23170839, 19163425, 19550195
     20389257, 18849021, 18861196, 17940721, 18524834, 19287706, 19522313
     18835283, 18748932, 19184765, 21639478, 14851828, 20598625, 19226141
     19053891, 18871287, 25371730, 18998228, 18980002, 19682778, 19325604
     8934841, 18777835, 17338864, 18945906, 19513650, 19702758, 18952577
     19473088, 19178517, 18729269, 19529729, 19455563, 20877072, 23275930
     20340620, 18536826, 19292605, 22264951, 20920711, 20660273, 20011635
     17592037, 18935857, 22989069, 19029647, 19179158, 22302421, 23076471
     19140712, 20012766, 23049301, 19184276, 25206643, 20510208, 20001507
     20079414, 19680763, 19259765, 18962892, 19513888, 14661880, 21519340
     21222147, 20218012, 19551830, 21616278, 19068333, 18945435, 20775015
     19319192, 20839581, 19178629, 18774591, 21694237, 19304104, 19284357
     22594310, 20725084, 19457575, 21297657, 19069755, 19021575, 18715884
     19584688, 18798573, 19812592, 19018001, 19292272, 19270956, 21967791
     21794779, 23055224, 18700893, 21276514, 20467377, 18406774, 20607821
     21976167, 21082737, 21694632, 20115586, 20883009, 19164099, 18283792
     19155350, 19884033, 19146822, 18798432, 21123424, 24356071, 19471836
     20936562, 18750781, 19045143, 20559126, 22483804, 19146980, 19601003
     20099624, 21378016, 19537547, 19281106, 18862203, 19954372, 18235274
     19205617, 19513351, 20219458, 19150313, 18843054, 18708349, 19332386
     18953639, 21423055, 19371270, 25221056, 19203996, 20038431, 19370846
     21053036, 18998379, 20157569, 21357522, 19273577, 24709351, 25206634
     22113170, 19632437, 18990354, 19288396, 20620581, 19557558, 19127078
     22548137, 21792881, 20900588, 20053557, 22088198, 22664609, 19315567
     19444471, 19154933, 20953798, 18290252, 18813323, 19915945, 19500293
     18643483, 19134098, 23034352, 25206614, 18965694, 19226858, 19591356
     19602208, 22884308, 20069197, 18417590, 18920408, 18263259, 18636884
     22289350, 19043795, 20746507, 18260170, 18919682, 18678829, 19319357
     19273758, 18953878, 18999195, 24314323, 18759724, 20458032, 19066844
     13843841, 20290467, 22934855, 19649640, 19062675, 21924910, 18859710
     20862812, 19504641, 23101611, 19341481, 20293730, 19343245, 18304090
     19241655, 19470791, 18242738, 20649573, 18372060, 18953889, 18827679
     19140711, 19023430, 19259290, 19045388, 20186278, 25197395, 19522571
     19709663, 18875012, 18861564, 21075392, 19273760, 19225265, 15885689
     20853464, 21696068, 18819158, 21394146, 19049721, 19368917, 25206650
     21255373, 22393909, 20408163, 20579351, 18868829, 20350915, 21327402
     22024217, 23101338, 19820247, 25206585, 20363708, 19538241, 19804032

Patch  25171037     : applied on Sat May 20 00:41:36 IST 2017
Unique Patch ID:  21099266
Patch description:  "Database Patch Set Update : 12.1.0.2.170418 (25171037)"
   Created on 7 Mar 2017, 12:37:23 hrs PST8PDT
Sub-patch  24732082; "Database Patch Set Update : 12.1.0.2.170117 (24732082)"
Sub-patch  24006101; "Database Patch Set Update : 12.1.0.2.161018 (24006101)"
Sub-patch  23054246; "Database Patch Set Update : 12.1.0.2.160719 (23054246)"
Sub-patch  22291127; "Database Patch Set Update : 12.1.0.2.160419 (22291127)"
Sub-patch  21948354; "Database Patch Set Update : 12.1.0.2.160119 (21948354)"
Sub-patch  21359755; "Database Patch Set Update : 12.1.0.2.5 (21359755)"
Sub-patch  20831110; "Database Patch Set Update : 12.1.0.2.4 (20831110)"
Sub-patch  20299023; "Database Patch Set Update : 12.1.0.2.3 (20299023)"
Sub-patch  19769480; "Database Patch Set Update : 12.1.0.2.2 (19769480)"
   Bugs fixed:
     21099555, 22175564, 19141838, 22083366, 20842388, 20117253, 19865345
     19791273, 21542577, 20951038, 19243521, 22165897, 17008068, 19908836
     21281532, 19238590, 24577566, 21184223, 19134173, 20569094, 20031873
     20322560, 20387265, 21575362, 19149990, 21263635, 17551063, 18886413
     22160989, 22507210, 19366375, 19703301, 19001390, 24285405, 18202441
     20267166, 19358317, 19706965, 19068970, 18549238, 24739928, 18797519
     22148226, 20825533, 21196809, 19649152, 19670108, 18940497, 18948177
     22496904, 18964978, 19176326, 19035573, 20413820, 19176223, 21106027
     20904530, 20134339, 19074147, 20868862, 18411216, 25475853, 21322887
     21072646, 22507234, 20425790, 20862087, 18966843, 21329301, 20562898
     19333670, 20124446, 19468991, 19883092, 20878790, 18510194, 19658708
     19591608, 19402853, 20618595, 21787056, 22380919, 19469538, 21266085
     17835294, 19721304, 19068610, 19791377, 22178855, 16777441, 22173980
     20048359, 20746251, 21896069, 19185876, 20898391, 20907061, 20281121
     6599380, 19577410, 22092979, 19001359, 20603378, 23089357, 21387964
     19490948, 22294260, 17532734, 20832516, 22351572, 19309466, 20627866
     19081128, 20844426, 21188532, 18791688, 20890311, 21442094, 20596234
     20368850, 18973548, 19303936, 21296029, 20882568, 19461270, 21479753
     22077517, 20936905, 20235511, 21220620, 18964939, 19430401, 22296366
     21153266, 19409212, 20703000, 22657942, 19879746, 20657441, 21294938
     19684504, 19024808, 20528052, 24693382, 20977794, 18799993, 20466322
     18740837, 19662635, 18440095, 20228093, 19065556, 20212067, 21868720
     22905130, 19524384, 24350831, 17722075, 20446883, 25056052, 18952989
     24523374, 16870214, 19928926, 19835133, 21629064, 21354456, 20466628
     24386767, 25490238, 19931709, 19730508, 18819908, 20250147, 23124895
     23220453, 19188927, 20074391, 18307021, 20356733, 14643995, 19065677
     19547370, 21960504, 21225209, 20397490, 18967382, 19174430, 21241829
     19536415, 19171086, 21889720, 22465352, 22168163, 19335438, 24397438
     20447445, 18856999, 19869255, 20471920, 21620471, 23096938, 18990693
     19124336, 17890099, 24812585, 18990023, 21300341, 20101006, 20848335
     21744290, 20897759, 21668627, 19304354, 20543011, 19052488, 20794034
     23025340, 23260854, 18681056, 20952966, 19896336, 25539063, 18618122
     20328248, 20440930, 18456643, 19699191, 19201867, 22865673, 22022760
     20798891, 18743542, 25161298, 20347562, 22551446, 19777862, 19687159
     21373076, 19174942, 20424899, 21899588, 18899974, 21476308, 20598042
     21297872, 24308635, 20171986, 19058490, 19032777, 22815955, 19399918
     19434529, 21273804, 19018447, 22757364, 18851894, 19022470, 19284031
     18043064, 20173897, 22062026, 20475845, 17274537, 19440586, 18974476
     24825843, 22374754, 16887946, 17319928, 20401975, 20708701, 22062517
     22809871, 17655240, 16439813, 19805359, 19155797, 20859910, 19393542
     22024071, 17210525, 19189525, 21847223, 21649497, 25079710, 19075256
     20315311, 22762046, 22075064, 20936731, 18845653, 19280225, 19248799
     20560611, 18988834, 21756699, 18921743, 20245930, 18799063, 20373598
     19571367, 20476175, 20925795, 19018206, 25264559, 20711718, 20509482
     20181030, 20588502, 21911701, 18849537, 23501901, 19183343, 21917884
     21142837, 19189317, 19644859, 19390567, 19279273, 20669434, 16863642
     22528741, 25546608, 19619732, 18607546, 20348653, 19315691, 19676905
     20165574, 17867700, 20558005, 20734332, 19532017, 20922010, 19818513
     19450314, 22353346, 16941434, 20361671, 20009833, 22366558, 20294666
     18191823, 23197103, 19195895, 19371175, 19307662, 19154375, 20043616
     21977392, 18914624, 22529728, 25330273, 20139391, 19593445, 21291274
     19382851, 19520602, 19174521, 21875360, 19676012, 19326908, 20217801
     20093776, 18840932, 21097043, 21246723, 20803014, 21665897, 19143550
     20428621, 19627012, 14283239, 21422580, 19213447, 19518079, 18610915
     18674024, 24413809, 18306996, 19915271, 19524158, 20122715, 20284155
     20017509, 22359063, 19363645, 19597439, 21239530, 19383839, 20880215
     21756677, 19888853, 22458049, 19534363, 19354335, 19044962, 19639483
     19475971, 22353199, 22243719, 21060755, 22916353, 20378086, 24808595
     21756661, 21260431, 22923409, 19028800, 20877664, 21059919, 20879889
     21380789, 19723336, 19077215, 19604659, 21421886, 21285458, 23533524
     23170620, 22365117, 18288842, 19048007, 19308965, 19689979, 19503821
     21526048, 19197175, 19180770, 19902195, 23324000, 20318889, 19013183
     20591183, 19012119, 20464614, 19067244, 21632821, 19841800, 19512341
     22695831, 20331945, 19587324, 24316947, 19578350, 19637186, 19054077
     18674047, 19708632, 20898997, 21091431, 19289642, 21133343, 20869721
     21172913, 19258504, 17365043, 21419850, 19468347, 21373473, 25093739
     16359751, 21164318, 22520320, 19769480, 19439759, 19272708, 19978542
     19329654, 20402832, 19873610, 23229229, 13542050, 21517440, 19291380
     21915719, 20879709, 20677396, 19076343, 19561643, 19990037, 19487147
     18909599, 20831538, 19016730, 18250893, 16619249, 18354830, 24411921
     16756406, 18254023, 21188584, 19989009, 17414008, 20688221, 20704450
     20441797, 25483815, 19157754, 24437510, 18885870, 21785691, 20673810
     21450666, 18893947, 18705806, 22223463, 16923858, 18417036, 23314180
     20919320, 20474192, 22046677, 21299490, 19501299, 19385656, 20920911
     20899461, 21387128, 21315084, 18122373, 20581111, 19606174, 24690216
     18436647, 19023822, 19124589, 19178851, 19597583, 18499088, 19050649



Rac system comprising of multiple nodes
  Local node = myserver02
  Remote node = myserver01

--------------------------------------------------------------------------------

OPatch succeeded.
MYRPT2:/home/oracle>


[root@myserver01 bin]#
[root@myserver01 bin]# ./crsctl start crs
CRS-4123: Oracle High Availability Services has been started.
[root@myserver01 bin]#


[root@myserver02 ngetadmin]# cd /grid/12c/app/12.1.0.2/grid/bin/
[root@myserver02 bin]#
[root@myserver02 bin]# ./crsctl start crs
CRS-4123: Oracle High Availability Services has been started.
[root@myserver02 bin]#

[root@myserver01 bin]# ./crsctl stat res -t
--------------------------------------------------------------------------------
Name           Target  State        Server                   State details
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ARCH.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.BACKUP.BACKUP.advm
               ONLINE  ONLINE       myserver01               Volume device /dev/a
                                                             sm/backup-380 is onl
                                                             ine,STABLE
               ONLINE  ONLINE       myserver02               Volume device /dev/a
                                                             sm/backup-380 is onl
                                                             ine,STABLE
ora.BACKUP.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.CTRLDG1.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.CTRLDG2.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.CTRLDG3.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.DATA.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.DATA1.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.DATA2.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.LISTENER.lsnr
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.LISTENER_RDB.lsnr
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.OCRV.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.OGGDG.OGGVOL.advm
               ONLINE  ONLINE       myserver01               Volume device /dev/a
                                                             sm/oggvol-358 is onl
                                                             ine,STABLE
               ONLINE  ONLINE       myserver02               Volume device /dev/a
                                                             sm/oggvol-358 is onl
                                                             ine,STABLE
ora.OGGDG.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.REDOA.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.REDOB.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.REDOC.dg
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.asm
               ONLINE  ONLINE       myserver01               Started,STABLE
               ONLINE  ONLINE       myserver02               Started,STABLE
ora.backup.backup.acfs
               ONLINE  ONLINE       myserver01               mounted on /backup,S
                                                             TABLE
               ONLINE  ONLINE       myserver02               mounted on /backup,S
                                                             TABLE
ora.net1.network
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
ora.oggdg.oggvol.acfs
               ONLINE  ONLINE       myserver01               mounted on /ogg,STAB
                                                             LE
               ONLINE  ONLINE       myserver02               mounted on /ogg,STAB
                                                             LE
ora.ons
               ONLINE  ONLINE       myserver01               STABLE
               ONLINE  ONLINE       myserver02               STABLE
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
      1        ONLINE  ONLINE       myserver02               STABLE
ora.LISTENER_SCAN2.lsnr
      1        ONLINE  ONLINE       myserver01               STABLE
ora.LISTENER_SCAN3.lsnr
      1        ONLINE  ONLINE       myserver01               STABLE
ora.MGMTLSNR
      1        ONLINE  ONLINE       myserver01               169.254.12.218 192.1
                                                             68.243.1 192.168.244
                                                             .1,STABLE
ora.cvu
      1        ONLINE  ONLINE       myserver01               STABLE
ora.mgmtdb
      1        ONLINE  ONLINE       myserver01               Open,STABLE
ora.myserver01.vip
      1        ONLINE  ONLINE       myserver01               STABLE
ora.myserver02.vip
      1        ONLINE  ONLINE       myserver02               STABLE
ora.oc4j
      1        ONLINE  ONLINE       myserver01               STABLE
ora.MYRPT.db
      1        OFFLINE OFFLINE                               Instance Shutdown,ST
                                                             ABLE
      2        OFFLINE OFFLINE                               Instance Shutdown,ST
                                                             ABLE
ora.scan1.vip
      1        ONLINE  ONLINE       myserver02               STABLE
ora.scan2.vip
      1        ONLINE  ONLINE       myserver01               STABLE
ora.scan3.vip
      1        ONLINE  ONLINE       myserver01               STABLE
--------------------------------------------------------------------------------
[root@myserver01 bin]#
[root@myserver01 bin]#
[root@myserver01 bin]# ./crsctl stat res -t | grep OFFLINE
      1        OFFLINE OFFLINE                               Instance Shutdown,ST
      2        OFFLINE OFFLINE                               Instance Shutdown,ST
[root@myserver01 bin]#


[root@myserver01 bin]# df -kh
Filesystem                                       Size  Used Avail Use% Mounted on
/dev/mapper/vg_dbsrvr1-lv_root                    29G   17G   11G  62% /
tmpfs                                            127G  2.8G  124G   3% /dev/shm
/dev/mapper/vg_dbsrvr2-lv_agent                   20G  1.1G   18G   6% /agent
/dev/mapper/36005076043c14c381a9f09ba1ad22492p2  488M   94M  369M  21% /boot
/dev/mapper/36005076043c14c381a9f09ba1ad22492p1  200M  256K  200M   1% /boot/efi
/dev/mapper/vg_dbsrvr1-lv_dump                    29G   12G   17G  42% /dump
/dev/mapper/vg_dbsrvr2-lv_grid                    48G   38G  8.2G  83% /grid
/dev/mapper/vg_dbsrvr1-lv_home                    20G  8.0G   11G  44% /home
/dev/mapper/vg_dbsrvr2-lv_oracle                  96G   41G   51G  45% /oracle
/dev/mapper/vg_dbsrvr1-lv_tmp                     20G   13G  6.1G  67% /tmp
/dev/mapper/vg_dbsrvr1-lv_var                    9.5G  4.5G  4.6G  50% /var
/dev/asm/oggvol-358                               99G   30G   70G  30% /ogg
/dev/asm/backup-380                              2.0T  1.2T  837G  59% /backup
[root@myserver01 bin]#

[root@myserver02 bin]# df -kh
Filesystem                                       Size  Used Avail Use% Mounted on
/dev/mapper/vg_dbsrvr1-lv_root                    29G   13G   15G  46% /
tmpfs                                            127G  639M  126G   1% /dev/shm
/dev/mapper/vg_dbsrvr2-lv_agent                   20G  1.1G   18G   6% /agent
/dev/mapper/36005076043c14f781a9f0df1505843c8p2  488M   94M  369M  21% /boot
/dev/mapper/36005076043c14f781a9f0df1505843c8p1  200M  256K  200M   1% /boot/efi
/dev/mapper/vg_dbsrvr1-lv_dump                    29G  4.0G   24G  15% /dump
/dev/mapper/vg_dbsrvr2-lv_grid                    48G   40G  6.0G  88% /grid
/dev/mapper/vg_dbsrvr1-lv_home                    20G  4.0G   15G  22% /home
/dev/mapper/vg_dbsrvr2-lv_oracle                  96G   35G   57G  38% /oracle
/dev/mapper/vg_dbsrvr1-lv_tmp                     20G   57M   19G   1% /tmp
/dev/mapper/vg_dbsrvr1-lv_var                    9.5G  3.2G  5.9G  35% /var
/dev/asm/oggvol-358                               99G   30G   70G  30% /ogg
/dev/asm/backup-380                              2.0T  1.2T  837G  59% /backup
[root@myserver02 bin]#


MYRPT1:/home/oracle>srvctl status database -d MYRPT
Instance MYRPT1 is not running on node myserver01
Instance MYRPT2 is not running on node myserver02
MYRPT1:/home/oracle>srvctl start database -d MYRPT
MYRPT1:/home/oracle>
MYRPT1:/home/oracle>srvctl status database -d MYRPT
Instance MYRPT1 is running on node myserver01
Instance MYRPT2 is running on node myserver02
MYRPT1:/home/oracle>



No comments:

Post a Comment