iSharkFly-Open
/
fis-gtm
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
fis-gtm/sr_unix/gtmsource_process.c

1681 lines
79 KiB
C
Raw Blame History

/****************************************************************
* *
* Copyright 2006, 2012 Fidelity Information Services, Inc.*
* *
* This source code contains the intellectual property *
* of its copyright holder(s), and is made available *
* under a license. If you do not know the terms of *
* the license, please stop and do not read further. *
* *
****************************************************************/
#if defined(__MVS__) && !defined(_ISOC99_SOURCE)
#define _ISOC99_SOURCE
#endif
#include "mdef.h"
#include "gtm_string.h"
#include "gtm_stdio.h"
#include "gtm_socket.h"
#include "gtm_inet.h"
#include "gtm_fcntl.h"
#include "gtm_unistd.h"
#include "gtm_time.h"
#include "gtm_stat.h"
#include <sys/time.h>
#include <errno.h>
#include <signal.h>
#ifdef VMS
#include <descrip.h> /* Required for gtmsource.h */
#endif
#include "gdsroot.h"
#include "gdsblk.h"
#include "gtm_facility.h"
#include "fileinfo.h"
#include "gdsbt.h"
#include "gdsfhead.h"
#include "filestruct.h"
#include "repl_msg.h"
#include "gtmsource.h"
#include "repl_comm.h"
#include "jnl.h"
#include "hashtab_mname.h" /* needed for muprec.h */
#include "hashtab_int4.h" /* needed for muprec.h */
#include "hashtab_int8.h" /* needed for muprec.h */
#include "buddy_list.h"
#include "muprec.h"
#include "repl_ctl.h"
#include "repl_errno.h"
#include "repl_dbg.h"
#include "iosp.h"
#include "gt_timer.h"
#include "gtmsource_heartbeat.h"
#include "repl_filter.h"
#include "repl_log.h"
#include "min_max.h"
#include "rel_quant.h"
#include "copy.h"
#include "ftok_sems.h"
#include "repl_instance.h"
#include "gtmmsg.h"
#include "repl_sem.h"
#include "have_crit.h" /* needed for ZLIB_COMPRESS */
#include "deferred_signal_handler.h" /* needed for ZLIB_COMPRESS */
#include "gtm_zlib.h"
#include "repl_sort_tr_buff.h"
#include "replgbl.h"
#include "gtmsource_srv_latch.h"
#include "gv_trigger_common.h"
#define MAX_HEXDUMP_CHARS_PER_LINE 26 /* 2 characters per byte + space, 80 column assumed */
#define BREAK_IF_CMP_ERROR(CMPRET, SEND_TR_LEN) \
{ \
switch(CMPRET) \
{ \
case Z_MEM_ERROR: \
repl_log(gtmsource_log_fp, TRUE, FALSE, "Out-of-memory error from compress function " \
"while compressing %d bytes\n", SEND_TR_LEN); \
assert(FALSE); \
break; \
case Z_BUF_ERROR: \
repl_log(gtmsource_log_fp, TRUE, FALSE, "Insufficient output buffer error from compress function " \
"while compressing %d bytes\n", SEND_TR_LEN); \
assert(FALSE); \
break; \
case Z_STREAM_ERROR: \
repl_log(gtmsource_log_fp, TRUE, FALSE, "Compression level %d invalid error from compress function " \
"while compressing %d bytes\n", repl_zlib_cmp_level, SEND_TR_LEN); \
assert(FALSE); \
break; \
} \
}
#define SET_8BYTE_CMP_MSGHDR(SEND_MSGP, SEND_TR_LEN, CMPBUFLEN, MSGHDRLEN) \
{ \
SEND_MSGP->type = (SEND_TR_LEN << REPL_TR_CMP_MSG_TYPE_BITS) | REPL_TR_CMP_JNL_RECS; \
SEND_MSGP->len = (int4)cmpbuflen + msghdrlen; \
/* Note that a compressed message need not be 8-byte aligned even though the input message was. So round it up to \
* the nearest align boundary. The actual message will contain the unaligned length which is what the receiver will \
* receive. But the # of bytes transmitted across will be the aligned length. \
*/ \
SEND_TR_LEN = ROUND_UP(SEND_MSGP->len, REPL_MSG_ALIGN); \
}
#define SET_16BYTE_CMP_MSGHDR(SEND_MSGP, SEND_TR_LEN, CMPBUFLEN, MSGHDRLEN) \
{ \
repl_cmpmsg_ptr_t send_cmpmsgp; \
\
send_cmpmsgp = (repl_cmpmsg_ptr_t)SEND_MSGP; \
assert(&send_cmpmsgp->type == &SEND_MSGP->type); \
assert(&send_cmpmsgp->len == &SEND_MSGP->len); \
send_cmpmsgp->type = REPL_TR_CMP_JNL_RECS2; \
/* Note that a compressed message need not be 8-byte aligned even though the input message was. So round it up to \
* the nearest align boundary. The actual message will contain the unaligned length which is what the receiver will \
* receive. But the # of bytes transmitted across will be the aligned length. \
*/ \
send_cmpmsgp->len = (int4)(ROUND_UP(CMPBUFLEN + MSGHDRLEN, REPL_MSG_ALIGN)); \
send_cmpmsgp->uncmplen = SEND_TR_LEN; \
send_cmpmsgp->cmplen = (int4)CMPBUFLEN; \
SEND_TR_LEN = SEND_MSGP->len; \
}
#ifdef GTM_TRIGGER
#define ISSUE_TRIG2NOTRIG_IF_NEEDED \
{ \
DCL_THREADGBL_ACCESS; \
\
SETUP_THREADGBL_ACCESS; \
if (!(TREF(replgbl)).trig_replic_warning_issued && (TREF(replgbl)).trig_replic_suspect_seqno \
&& !remote_side->trigger_supported) \
{ /* Note: The below repl_log text is copied from TRIG2NOTRIG error message content from merrors.msg. Change \
* to one should be reflected in another \
*/ \
repl_log(gtmsource_log_fp, TRUE, TRUE, "Warning: Sending transaction sequence number %d which used " \
"triggers to a replicator that does not support triggers\n", (TREF(replgbl)).trig_replic_suspect_seqno);\
(TREF(replgbl)).trig_replic_warning_issued = TRUE; /* No more warnings till restart */ \
(TREF(replgbl)).trig_replic_suspect_seqno = seq_num_zero; \
} \
}
#endif
GBLDEF struct timeval gtmsource_poll_wait, gtmsource_poll_immediate;
GBLDEF repl_msg_ptr_t gtmsource_msgp = NULL;
GBLDEF int gtmsource_msgbufsiz = 0;
GBLDEF repl_msg_ptr_t gtmsource_cmpmsgp = NULL;
GBLDEF int gtmsource_cmpmsgbufsiz = 0;
GBLDEF boolean_t gtmsource_received_cmp2uncmp_msg;
GBLDEF qw_num repl_source_data_sent = 0;
GBLDEF qw_num repl_source_msg_sent = 0;
GBLDEF qw_num repl_source_cmp_sent = 0;
GBLDEF qw_num repl_source_lastlog_data_sent = 0;
GBLDEF qw_num repl_source_lastlog_msg_sent = 0;
GBLDEF time_t repl_source_prev_log_time;
GBLDEF time_t repl_source_this_log_time;
GBLDEF time_t gtmsource_last_flush_time;
GBLREF gtmsource_state_t gtmsource_state;
GBLREF uchar_ptr_t repl_filter_buff;
GBLREF int repl_filter_bufsiz;
GBLREF volatile time_t gtmsource_now;
GBLREF int gtmsource_sock_fd;
GBLREF jnlpool_addrs jnlpool;
GBLREF gd_addr *gd_header;
GBLREF sgmnt_addrs *cs_addrs;
GBLREF sgmnt_data_ptr_t cs_data;
GBLREF gd_region *gv_cur_region;
GBLREF repl_ctl_element *repl_ctl_list;
GBLREF gtmsource_options_t gtmsource_options;
GBLREF int gtmsource_log_fd;
GBLREF int gtmsource_statslog_fd;
GBLREF FILE *gtmsource_log_fp;
GBLREF FILE *gtmsource_statslog_fp;
GBLREF boolean_t gtmsource_logstats;
GBLREF int gtmsource_filter;
GBLREF gd_addr *gd_header;
GBLREF seq_num seq_num_zero, seq_num_minus_one, seq_num_one;
GBLREF unsigned int jnl_source_datalen, jnl_dest_maxdatalen;
GBLREF unsigned char jnl_source_rectype, jnl_dest_maxrectype;
GBLREF int repl_max_send_buffsize, repl_max_recv_buffsize;
GBLREF seq_num lastlog_seqno;
GBLREF uint4 log_interval;
GBLREF qw_num trans_sent_cnt, last_log_tr_sent_cnt;
GBLREF repl_conn_info_t *this_side, *remote_side;
GBLREF int4 strm_index;
GBLREF uint4 process_id;
GBLREF seq_num gtmsource_save_read_jnl_seqno;
error_def(ERR_JNLNEWREC);
error_def(ERR_JNLSETDATA2LONG);
error_def(ERR_REPLCOMM);
error_def(ERR_REPLFTOKSEM);
error_def(ERR_REPLGBL2LONG);
error_def(ERR_REPLINSTNOHIST);
error_def(ERR_REPLNOMULTILINETRG);
error_def(ERR_REPLRECFMT);
error_def(ERR_REPLXENDIANFAIL);
error_def(ERR_SECNODZTRIGINTP);
error_def(ERR_TRIG2NOTRIG);
error_def(ERR_TEXT);
/* Endian converts the given set of journal records (possibly multiple sequence numbers) so that the secondary can consume them
* as-is. This is done only in the case when the primary is running on a GT.M version less than the GT.M version on secondary
* side. Otherwise, the secondary takes the responsibility of doing the endian conversion. Note that the endian conversion happens
* in-place. The below function is based on gtmrecv_process.c/repl_tr_endian_convert()
*/
static void repl_tr_endian_convert(repl_msg_ptr_t send_msgp, int send_tr_len, seq_num pre_read_seqno)
{
uchar_ptr_t buffp, jb;
DEBUG_ONLY(uchar_ptr_t jstart;)
int buflen, remaining_len, jlen, reclen, status, nodeflags_keylen, temp_val, keylen;
jnl_record *rec;
enum jnl_record_type rectype;
jrec_suffix *suffixp;
jnl_string *keystr;
mstr_len_t *vallen_ptr;
/* seq_num good_seqno; */
buffp = send_msgp->msg;
buflen = send_msgp->len - REPL_MSG_HDRLEN;
remaining_len = send_tr_len;
/* QWASSIGN(good_seqno, seq_num_zero); */
while (0 < remaining_len)
{
jlen = buflen;
jb = buffp;
while (JREC_PREFIX_SIZE <= jlen)
{
DEBUG_ONLY(jstart = jb);
rec = (jnl_record *)(jb);
/* endian convert the prefix fields. Not all of the prefix fields are used by the secondary. Only rectype
* and forwptr are needed.
*/
rectype = (enum jnl_record_type)rec->prefix.jrec_type;
reclen = rec->prefix.forwptr;
rec->prefix.forwptr = GTM_BYTESWAP_24(reclen);
if (!IS_REPLICATED(rectype) || (0 == reclen) || (reclen > jlen) || (reclen > MAX_LOGI_JNL_REC_SIZE))
{
assert(FALSE);
status = -1;
break;
}
assert(!IS_ZTP(rectype));
assert(IS_SET_KILL_ZKILL_ZTRIG_ZTWORM(rectype) || (JRT_TCOM == rectype) || (JRT_NULL == rectype));
/* endian convert the suffix fields. Only backptr needs endian conversion as the other field - suffix_code
* is 8 bit.
*/
suffixp = ((jrec_suffix *)((unsigned char *)rec + reclen - JREC_SUFFIX_SIZE));
suffixp->backptr = GTM_BYTESWAP_24(suffixp->backptr);
/* QWASSIGN(good_seqno, rec->jrec_null.jnl_seqno); */ /* update good_seqno */
rec->jrec_null.jnl_seqno = GTM_BYTESWAP_64(rec->jrec_null.jnl_seqno);
/* At this point, we could have a TCOM or NULL or SET/KILL/ZKILL/ZTRIG type of record.
* Assert that all of them have "strm_seqno" at the exact same offset so we can avoid
* an if/then/else check on the record types in order to endian convert "strm_seqno".
*/
assert(&rec->jrec_null.strm_seqno == &rec->jrec_set_kill.strm_seqno);
assert(&rec->jrec_null.strm_seqno == &rec->jrec_tcom.strm_seqno);
rec->jrec_null.strm_seqno = GTM_BYTESWAP_64(rec->jrec_null.strm_seqno);
if (IS_SET_KILL_ZKILL_ZTRIG_ZTWORM(rectype))
{
keystr = (jnl_string *)&rec->jrec_set_kill.mumps_node;
assert(keystr == (jnl_string *)&rec->jrec_ztworm.ztworm_str);
assert(&rec->jrec_set_kill.update_num == &rec->jrec_ztworm.update_num);
rec->jrec_set_kill.update_num = GTM_BYTESWAP_32(rec->jrec_set_kill.update_num);
/* From V19 onwards, the 'length' field is divided into 8 bit 'nodeflags' and 24 bit 'length'
* fields.
*/
keylen = keystr->length;
nodeflags_keylen = *(jnl_str_len_t *)keystr;
*(jnl_str_len_t *)keystr = GTM_BYTESWAP_32(nodeflags_keylen);
if (IS_SET(rectype) || IS_ZTWORM(rectype))
{ /* SET and ZTWORM records have a 'value' part which needs to be endian converted */
vallen_ptr = (mstr_len_t *)&keystr->text[keylen];
GET_MSTR_LEN(temp_val, vallen_ptr);
temp_val = GTM_BYTESWAP_32(temp_val);
PUT_MSTR_LEN(vallen_ptr, temp_val);
/* The actual 'value' itself is a character array and hence needs no endian conversion */
}
} else if (JRT_TCOM == rectype)
{
assert((unsigned char *)&rec->jrec_tcom.token_seq
+ SIZEOF(token_seq_t) == (unsigned char *)&rec->jrec_tcom.filler_short);
/* endian convert num_participants */
rec->jrec_tcom.num_participants = GTM_BYTESWAP_16(rec->jrec_tcom.num_participants);
}
/* else records can only be JRT_NULL. The only relevant field in JRT_NULL is the sequence number which is
* already endian converted.
*/
assert(jstart == jb); /* endian conversion should always happen in-place. */
jlen -= reclen;
jb += reclen;
}
if ((-1 == status) || (0 != jlen))
{
assert(FALSE);
rts_error(VARLSTCNT(5) ERR_REPLXENDIANFAIL, 3, LEN_AND_LIT("Originating"), &pre_read_seqno);
}
/* move on to the next transaction */
remaining_len -= (buflen + REPL_MSG_HDRLEN);
buffp += buflen;
assert((REPL_TR_JNL_RECS == ((repl_msg_ptr_t)(buffp))->type) || (0 == remaining_len));
buflen = ((repl_msg_ptr_t)(buffp))->len - REPL_MSG_HDRLEN;
buffp += REPL_MSG_HDRLEN;
}
if (0 != remaining_len)
{
rts_error(VARLSTCNT(5) ERR_REPLXENDIANFAIL, 3, LEN_AND_LIT("Originating"), &pre_read_seqno);
assert(FALSE);
}
}
/* The work-horse of the Source Server */
int gtmsource_process(void)
{
gtmsource_local_ptr_t gtmsource_local;
jnlpool_ctl_ptr_t jctl;
seq_num recvd_seqno, sav_read_jnl_seqno;
struct sockaddr_in secondary_addr;
seq_num recvd_jnl_seqno, tmp_read_jnl_seqno;
int data_len, srch_status;
unsigned char *msg_ptr; /* needed for REPL_{SEND,RECV}_LOOP */
int tosend_len, sent_len, sent_this_iter; /* needed for REPL_SEND_LOOP */
int torecv_len, recvd_len, recvd_this_iter; /* needed for REPL_RECV_LOOP */
int status; /* needed for REPL_{SEND,RECV}_LOOP */
int tot_tr_len, send_tr_len, remaining_len, pre_cmpmsglen;
struct timeval poll_time;
int recvd_msg_type, recvd_start_flags;
uchar_ptr_t in_buff, out_buff, out_buffmsg;
uint4 in_buflen, out_buflen, out_bufsiz;
seq_num log_seqno, diff_seqno, pre_read_seqno, post_read_seqno, jnl_seqno;
char err_string[1024];
boolean_t xon_wait_logged, prev_catchup, catchup, force_recv_check, already_communicated;
double time_elapsed;
seq_num resync_seqno, zqgblmod_seqno, filter_seqno;
gd_region *reg, *region_top;
sgmnt_addrs *csa;
qw_num backlog_bytes, backlog_count, delta_sent_cnt, delta_data_sent, delta_msg_sent;
long prev_msg_sent = 0;
time_t prev_now = 0, save_now;
int index;
struct timeval poll_wait, poll_immediate;
uint4 temp_ulong;
unix_db_info *udi;
repl_histinfo remote_histinfo, local_histinfo;
int4 num_histinfo, max_epoch_interval;
seq_num local_jnl_seqno, tmp_seqno;
repl_msg_t xoff_ack, instnohist_msg, losttncomplete_msg;
repl_msg_ptr_t send_msgp;
repl_cmpmsg_ptr_t send_cmpmsgp;
repl_start_reply_msg_ptr_t reply_msgp;
boolean_t rollback_first, secondary_ahead, secondary_was_rootprimary;
boolean_t intfilter_error, skip_last_histinfo_check, msg_is_cross_endian, retval;
int semval, cmpret;
uLongf cmpbuflen;
int4 msghdrlen;
Bytef *cmpbufptr;
char histdetail[256];
gtm_time4_t tmp_time4;
repl_heartbeat_msg_ptr_t heartbeat_msg;
DEBUG_ONLY(uchar_ptr_t save_inbuff;)
DEBUG_ONLY(uchar_ptr_t save_outbuff;)
DCL_THREADGBL_ACCESS;
SETUP_THREADGBL_ACCESS;
assert((NULL != jnlpool.jnlpool_dummy_reg) && jnlpool.jnlpool_dummy_reg->open);
DEBUG_ONLY(
csa = &FILE_INFO(jnlpool.jnlpool_dummy_reg)->s_addrs;
assert(!csa->hold_onto_crit); /* so we can do unconditional grab_lock/rel_lock */
ASSERT_VALID_JNLPOOL(csa);
)
assert(REPL_MSG_HDRLEN == SIZEOF(jnldata_hdr_struct)); /* necessary for reading multiple transactions from jnlpool in
* a single attempt */
jctl = jnlpool.jnlpool_ctl;
gtmsource_local = jnlpool.gtmsource_local;
gtmsource_msgp = NULL;
gtmsource_msgbufsiz = MAX_REPL_MSGLEN;
if (ZLIB_CMPLVL_NONE != gtm_zlib_cmp_level)
gtmsource_cmpmsgp = NULL;
assert(GTMSOURCE_POLL_WAIT < MAX_GTMSOURCE_POLL_WAIT);
gtmsource_poll_wait.tv_sec = 0;
gtmsource_poll_wait.tv_usec = GTMSOURCE_POLL_WAIT;
poll_wait = gtmsource_poll_wait;
gtmsource_poll_immediate.tv_sec = 0;
gtmsource_poll_immediate.tv_usec = 0;
poll_immediate = gtmsource_poll_immediate;
gtmsource_init_sec_addr(&secondary_addr);
gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION;
/* Below is a simplistic representation of the state diagram of a source server.
*
* ------------------------------
* GTMSOURCE_START
* ------------------------------
* |
* | (startup state)
* v
* ------------------------------
* GTMSOURCE_WAITING_FOR_CONNECTION
* ------------------------------
* |
* | (gtmsource_est_conn)
* v
* ------------------------------
* GTMSOURCE_WAITING_FOR_RESTART
* ------------------------------
* |
* | (gtmsource_recv_restart)
* v
* ------------------------------
* GTMSOURCE_SEARCHING_FOR_RESTART
* ------------------------------
* |
* | (gtmsource_srch_restart)
* v
* ------------------------------
* GTMSOURCE_SENDING_JNLRECS <---------\
* ------------------------------ |
* | |
* | (receive REPL_XOFF) |
* | (receive REPL_XOFF_ACK_ME) |
* v |
* ------------------------------ ^
* GTMSOURCE_WAITING_FOR_XON |
* ------------------------------ |
* | |
* v (receive REPL_XON) |
* | |
* \--------------------->---------/
*/
udi = FILE_INFO(jnlpool.jnlpool_dummy_reg);
/* Before entering the loop find the max EPOCH interval (for use in lock waits) */
max_epoch_interval = 0;
for (reg = gd_header->regions, region_top = gd_header->regions + gd_header->n_regions; reg < region_top; reg++)
{
csa = &FILE_INFO(reg)->s_addrs;
if (max_epoch_interval < csa->hdr->epoch_interval)
max_epoch_interval = csa->hdr->epoch_interval;
}
while (TRUE)
{
assert(!udi->grabbed_ftok_sem);
gtmsource_stop_heartbeat();
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
{
/* Ensure we don't hold any locks at this moment */
assert(process_id != gtmsource_local->gtmsource_srv_latch.u.parts.latch_pid);
assert(!have_crit(CRIT_HAVE_ANY_REG)); /* checks both journal pool lock and database crit lock */
assert(FD_INVALID != gtmsource_sock_fd);
if (FD_INVALID != gtmsource_sock_fd)
{
repl_log(gtmsource_log_fp, TRUE, TRUE, "Closing connection due to ONLINE ROLLBACK\n");
repl_close(&gtmsource_sock_fd);
SHORT_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_CLOSE_CONN);
}
jnl_seqno = jnlpool.jnlpool_ctl->jnl_seqno;
repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL INFO - Current Jnlpool Seqno : %llu\n",
jnl_seqno);
repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL INFO - Last Seqno sent : %llu\n",
gtmsource_local->read_jnl_seqno - 1);
/* gtmsource_save_read_jnl_seqno is kept uptodate with gtmsource_local->read_addr and gtmsource_local->read
* fields in gtmsource_onln_rlbk_clnup. But, gtmsource_local->read_jnl_seqno is still pointing to the last
* sequence number that we sent to the receiver (which could have been rolled back now). We don't want to
* continue with a stale value of read_jnl_seqno. So, set it to gtmsource_save_read_jnl_seqno which itself
* is taken from the jnlpool_ctl->jnl_seqno right when we detected the online rollback. We could have set
* this right when we set gtmsource_save_read_jnl_seqno but we don't do that because we want to print the
* old value in the log file but we can't use repl_log/gtmsource_log_fp in gtmsource_onln_rlbk_clnup() as
* it is bundled up as part of libgtmshr.so whereas repl_log is bundled in libmupip.a.
*/
gtmsource_local->read_jnl_seqno = gtmsource_save_read_jnl_seqno;
repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL INFO - Source Server Read Seqno is now set to : %llu\n",
gtmsource_local->read_jnl_seqno);
gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION;
assert(READ_FILE == gtmsource_local->read_state);
gtmsource_ctl_close(); /* can't rely on the journal files anymore since rollback could have touched them */
}
if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
{
gtmsource_start_jnl_release_timer();
gtmsource_est_conn(&secondary_addr);
gtmsource_stop_jnl_release_timer();
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
return (SS_NORMAL);
repl_source_data_sent = repl_source_msg_sent = repl_source_cmp_sent = 0;
repl_source_lastlog_data_sent = 0;
repl_source_lastlog_msg_sent = 0;
gtmsource_alloc_msgbuff(MAX_REPL_MSGLEN);
gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_RESTART;
recvd_start_flags = START_FLAG_NONE;
repl_source_prev_log_time = time(NULL);
}
if (GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state &&
SS_NORMAL != (status = gtmsource_recv_restart(&recvd_seqno, &recvd_msg_type, &recvd_start_flags)))
{
if (EREPL_RECV == repl_errno)
{
if (REPL_CONN_RESET(status))
{ /* Connection reset */
repl_log(gtmsource_log_fp, TRUE, TRUE,
"Connection reset while receiving restart SEQNO. Status = %d ; %s\n",
status, STRERROR(status));
repl_close(&gtmsource_sock_fd);
SHORT_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_CLOSE_CONN);
gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION;
continue;
} else
{
SNPRINTF(err_string, SIZEOF(err_string),
"Error receiving RESTART SEQNO. Error in recv : %s", STRERROR(status));
rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_STRING(err_string));
}
} else if (EREPL_SEND == repl_errno)
{
if (REPL_CONN_RESET(status))
{
repl_log(gtmsource_log_fp, TRUE, TRUE,
"Connection reset while sending XOFF_ACK due to possible update process shutdown. "
"Status = %d ; %s\n", status, STRERROR(status));
repl_close(&gtmsource_sock_fd);
SHORT_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_CLOSE_CONN);
gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION;
continue;
}
SNPRINTF(err_string, SIZEOF(err_string), "Error sending XOFF_ACK_ME message. Error in send : %s",
STRERROR(status));
rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_STRING(err_string));
} else if (EREPL_SELECT == repl_errno)
{
SNPRINTF(err_string, SIZEOF(err_string), "Error receiving RESTART SEQNO/sending XOFF_ACK_ME. "
"Error in select : %s", STRERROR(status));
rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_STRING(err_string));
}
}
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
return (SS_NORMAL);
/* Connection might have been closed if "gtmsource_recv_restart" got an unexpected message. In that case
* re-establish the same by continuing to the beginning of this loop. */
if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
continue;
assert(REPL_PROTO_VER_MULTISITE <= remote_side->proto_ver);
assert((GTMSOURCE_SEARCHING_FOR_RESTART == gtmsource_state) || (GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state));
/* Receiver runs on a version of GT.M that supports multi-site capability */
/* If gtmsource_state == GTMSOURCE_SEARCHING_FOR_RESTART, we have already communicated with the
* receiver and hence checked the instance info so no need to do it again.
*/
if (GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state)
{ /* Get replication instance info */
DEBUG_ONLY(secondary_was_rootprimary = -1;)
/* Note: As part of the REPL_INSTINFO message, the receiver could be sending a non-zero "strm_jnl_seqno"
* in some cases. If so, it will override "recvd_seqno" we saw before in the REPL_START_JNL_SEQNO message.
*/
if (!gtmsource_get_instance_info(&secondary_was_rootprimary, &recvd_seqno))
{
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
return (SS_NORMAL);
else if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
continue;
else
{ /* Got a REPL_XOFF_ACK_ME from the receiver. Restart the initial handshake */
assert(GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state);
continue;
}
}
assert((FALSE == secondary_was_rootprimary) || (TRUE == secondary_was_rootprimary));
}
rollback_first = FALSE;
secondary_ahead = FALSE;
GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
continue;
local_jnl_seqno = jctl->jnl_seqno;
rel_lock(jnlpool.jnlpool_dummy_reg);
/* Take care to set the flush parameter in repl_log calls below to FALSE until at least the first message
* gets sent back. This is so the fetchresync rollback on the other side does not timeout before receiving
* a response. */
assert(0 == GET_STRM_INDEX(recvd_seqno));
assert(0 == GET_STRM_INDEX(local_jnl_seqno));
repl_log(gtmsource_log_fp, TRUE, FALSE, "Current Journal Seqno of the instance is %llu [0x%llx]\n",
local_jnl_seqno, local_jnl_seqno);
if (recvd_seqno > local_jnl_seqno)
{ /* Secondary journal seqno is greater than that of the Primary. We know it is ahead of the primary. */
secondary_ahead = TRUE;
repl_log(gtmsource_log_fp, TRUE, FALSE,
"Secondary instance journal seqno %llu [0x%llx] is greater than Primary "
"instance journal seqno %llu [0x%llx]\n",
recvd_seqno, recvd_seqno, local_jnl_seqno, local_jnl_seqno);
/* Since the secondary is at least multi-site, the determination of the rollback seqno involves comparing
* the histinfo records between the primary and secondary starting down from "local_jnl_seqno-1"
* (done below). In either case, the secondary has to roll back to at most "local_jnl_seqno".
* Reset "recvd_seqno" to this number given that we have already recorded that the secondary is
* ahead of the primary.
*/
recvd_seqno = local_jnl_seqno;
}
/* Before setting "next_histinfo_seqno", check if we have at least one histinfo record in the replication instance
* file. The only case when there can be no histinfo records is if this instance is a propagating primary. Assert
* that. In this case, wait for this instance's primary to send the first histinfo record before setting the
* next_histinfo_seqno. Note that we are fetching the value of "num_histinfo" without holding a lock on the instance
* file but that is ok since all we care about is if it is 0 or not. We do not rely on the actual value.
*/
num_histinfo = jnlpool.repl_inst_filehdr->num_histinfo;
assert(0 <= num_histinfo);
assert(num_histinfo || jctl->upd_disabled);
gtmsource_local->next_histinfo_num = -1;/* Initial value. Reset by the call to "gtmsource_set_next_histinfo_seqno"
* invoked in turn by "gtmsource_send_new_histrec" down below */
if (jctl->upd_disabled && !num_histinfo)
{ /* Wait for corresponding primary to send a new histinfo record and the receiver server on this instance
* to write that to the replication instance file.
*/
assert(-1 == gtmsource_local->next_histinfo_num);
repl_log(gtmsource_log_fp, TRUE, TRUE, "Source server waiting for first history record to be written by "
"update process\n");
do
{
SHORT_SLEEP(GTMSOURCE_WAIT_FOR_FIRSTHISTINFO);
gtmsource_poll_actions(FALSE);
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
return (SS_NORMAL);
num_histinfo = jnlpool.repl_inst_filehdr->num_histinfo;
if (num_histinfo) /* Number of histinfos is non-zero */
break;
} while (TRUE);
repl_log(gtmsource_log_fp, TRUE, TRUE,
"First history record written by update process. Source server proceeding.\n");
}
/* Now get the latest histinfo record from the secondary. There are a few exceptions though.
* 1) If we came here because of a BAD_TRANS or CMP2UNCMP message from the receiver server.
* In this case, we have already been communicating with the receiver so no need to
* compare the histinfo record between primary and secondary.
* 2) If receiver server was started with -UPDATERESYNC and receiver is running pre-V55000.
* In this case there is no history record on the receiver side to compare against.
* In case the receiver is post-V55000, the -UPDATERESYNC would have required an instance
* file name as the value which would be used towards history record verification.
* 3) If receiver server was started with -UPDATERESYNC and receiver is >= V55000 and at a seqno
* which is EQUAL to the earliest seqno for which we have a history record on the primary.
* We have no way of verifying histories since we definitely dont have the history record
* for the receiver side seqno. Since -updateresync was used, assume they are in sync and
* start replicating from the earliest seqno for which we have a history record on the primary.
* 4) If recvd_seqno is 1. In this case, the receiver instance has been created afresh so its instance
* file is empty and we are guaranteed there is nothing to compare. So no point requesting it.
* Besides, this is a very common situation in practice that requiring -updateresync in this
* case seems user-unfriendly so we will let this one go by without a -updateresync particularly
* because there is no harm that can happen by allowing two such instances to connect/replicate.
*/
assert(0 != recvd_seqno);
if (1 == recvd_seqno)
skip_last_histinfo_check = TRUE;
else
{
if ((GTMSOURCE_WAITING_FOR_RESTART != gtmsource_state) && already_communicated)
skip_last_histinfo_check = TRUE;
else if (START_FLAG_UPDATERESYNC & recvd_start_flags)
{
repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL_START_JNL_SEQNO message has "
"START_FLAG_UPDATERESYNC bit set\n");
if (REPL_PROTO_VER_SUPPLEMENTARY > remote_side->proto_ver)
skip_last_histinfo_check = TRUE;
else
{
assert(jnlpool.repl_inst_filehdr->num_histinfo); /* should be at least 1 history record */
/* If -updateresync is specified and receiver instance seqno is exactly equal to the
* start_seqno of the earliest history record in the instance file, then skip last
* histinfo check. Note that in case both source and receiver instances are supplementary,
* we should be looking at the 0th stream only. Even in that case, we are guaranteed that
* the 0th history record in the instance file corresponds to the 0th stream. So it is
* safe to look at the start_seqno of just the 0th history record in all cases.
*/
GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
continue;
status = repl_inst_histinfo_get(0, &local_histinfo);
assert(0 == status); /* Since we pass histinfo_num of 0 which is >=0 and < num_histinfo */
rel_lock(jnlpool.jnlpool_dummy_reg);
if (local_histinfo.start_seqno == recvd_seqno)
skip_last_histinfo_check = TRUE;
else
skip_last_histinfo_check = FALSE;
}
} else
skip_last_histinfo_check = FALSE;
}
if (!skip_last_histinfo_check)
{ /* Find histinfo record in the local instance file corresponding to seqno "recvd_seqno-1" */
GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
continue;
assert(recvd_seqno <= local_jnl_seqno);
assert(recvd_seqno <= jctl->jnl_seqno);
assert((INVALID_SUPPL_STRM == strm_index) || (0 == strm_index));
status = repl_inst_histinfo_find_seqno(recvd_seqno, strm_index, &local_histinfo);
rel_lock(jnlpool.jnlpool_dummy_reg);
assert((0 != status) || (local_histinfo.start_seqno < recvd_seqno));
if (0 != status)
{ /* If recvd_seqno is the earliest history record's start_seqno and -udpateresync was
* specified, assume the two instances are in sync. Otherwise issue error and close connection.
* Send this error status to the receiver server before closing the connection.
* This way the receiver will know to shut down rather than loop back trying to
* reconnect. This avoids an infinite loop of connection open and closes
* between the source server and receiver server.
*/
assert(ERR_REPLINSTNOHIST == status); /* only error returned by "repl_inst_histinfo_find_seqno" */
assert((INVALID_HISTINFO_NUM == local_histinfo.histinfo_num)
|| (local_histinfo.start_seqno >= recvd_seqno));
if (!(START_FLAG_UPDATERESYNC & recvd_start_flags)
|| (INVALID_HISTINFO_NUM == local_histinfo.histinfo_num)
|| (local_histinfo.start_seqno > recvd_seqno))
{ /* recvd_seqno is PRIOR to the starting seqno of the instance file.
* In that case, issue error and close the connection.
*/
SPRINTF(histdetail, "seqno "INT8_FMT" "INT8_FMTX, recvd_seqno - 1, recvd_seqno - 1);
gtm_putmsg(VARLSTCNT(6) ERR_REPLINSTNOHIST, 4,
LEN_AND_STR(histdetail), LEN_AND_STR(udi->fn));
instnohist_msg.type = REPL_INST_NOHIST;
instnohist_msg.len = MIN_REPL_MSGLEN;
memset(&instnohist_msg.msg[0], 0, SIZEOF(instnohist_msg.msg));
gtmsource_repl_send((repl_msg_ptr_t)&instnohist_msg, "REPL_INST_NOHIST",
MAX_SEQNO, INVALID_SUPPL_STRM);
repl_log(gtmsource_log_fp, TRUE, TRUE,
"Connection reset due to above REPLINSTNOHIST error\n");
repl_close(&gtmsource_sock_fd);
SHORT_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_CLOSE_CONN);
gtmsource_state = gtmsource_local->gtmsource_state
= GTMSOURCE_WAITING_FOR_CONNECTION;
continue;
}
assert((0 == local_histinfo.histinfo_num) && (local_histinfo.start_seqno == recvd_seqno));
}
if (local_histinfo.start_seqno < recvd_seqno)
{
/* Find histinfo record in the remote instance file corresponding to seqno "recvd_seqno-1" */
retval = gtmsource_get_remote_histinfo(recvd_seqno, &remote_histinfo);
if (retval)
{
assert(remote_histinfo.start_seqno < recvd_seqno);
/* Check if primary and secondary have same histinfo for "recvd_seqno-1" */
rollback_first = !gtmsource_is_histinfo_identical(&remote_histinfo, &local_histinfo,
recvd_seqno, OK_TO_LOG_TRUE);
/* If local and remote sides are supplementary (i.e. P->Q replication), verify each
* stream level history as well. Do this only if the remote side is a receiver server
* (i.e. not rollback) and if we still intend on sending a REPL_WILL_RESTART_WITH_INFO
* message.
*/
assert(this_side->is_supplementary == jnlpool.repl_inst_filehdr->is_supplementary);
if (this_side->is_supplementary && remote_side->is_supplementary
&& (REPL_START_JNL_SEQNO == recvd_msg_type)
&& !rollback_first && !secondary_ahead)
retval = gtmsource_check_remote_strm_histinfo(recvd_seqno, &rollback_first);
}
if (!retval)
{
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
return (SS_NORMAL);
else if ((GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
|| (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state))
continue;
else
{ /* Got a REPL_XOFF_ACK_ME from receiver. Restart the initial handshake */
assert(GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state);
continue;
}
}
}
}
QWASSIGN(sav_read_jnl_seqno, gtmsource_local->read_jnl_seqno);
reply_msgp = (repl_start_reply_msg_ptr_t)gtmsource_msgp;
memset(reply_msgp, 0, SIZEOF(*reply_msgp)); /* to identify older releases in the future */
reply_msgp->len = MIN_REPL_MSGLEN;
reply_msgp->proto_ver = REPL_PROTO_VER_THIS;
reply_msgp->node_endianness = NODE_ENDIANNESS;
reply_msgp->is_supplementary = jnlpool.repl_inst_filehdr->is_supplementary;
assert((1 != recvd_seqno) || !rollback_first);
if ((GTMSOURCE_SEARCHING_FOR_RESTART == gtmsource_state) || (REPL_START_JNL_SEQNO == recvd_msg_type))
{
gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_SEARCHING_FOR_RESTART;
/* If the last histinfo record in both instances are NOT the same ("rollback_first" is TRUE)
* (possible only if the secondary is multi-site), or if secondary is ahead of the primary
* ("secondary_ahead" is TRUE) we do want the secondary to rollback first. Issue message to
* do rollback fetchresync. There is one exception though. And that is if -NORESYNC was
* specified on the receiver side. In this case, determine the resync/common point by comparing
* local and remote histinfo records from the tail of the instance file until we reach
* one seqno whose histinfo information is identical in both.
* Use this as the common point to send a REPL_WILL_RESTART_WITH_INFO message.
*/
if (!rollback_first && !secondary_ahead)
resync_seqno = recvd_seqno;
else if (START_FLAG_NORESYNC & recvd_start_flags)
{
repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL_START_JNL_SEQNO message has "
"START_FLAG_NORESYNC bit set\n");
assert(!skip_last_histinfo_check);
assert(1 != recvd_seqno);
if (!rollback_first)
{
assert(secondary_ahead);
assert(recvd_seqno == local_jnl_seqno);
/* The primary and secondary are in sync as of "recvd_seqno" the jnl seqno of the
* primary. So that is the common point. Send it across.
*/
resync_seqno = recvd_seqno;
} else
{
resync_seqno = gtmsource_find_resync_seqno(&local_histinfo, &remote_histinfo);
assert((MAX_SEQNO != resync_seqno) || (GTMSOURCE_CHANGING_MODE == gtmsource_state)
|| (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
|| (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state));
rollback_first = FALSE;
}
} else
{ /* Ask secondary to issue a fetchresync rollback */
repl_log(gtmsource_log_fp, TRUE, FALSE,
"Secondary instance needs to first do MUPIP JOURNAL ROLLBACK FETCHRESYNC\n");
resync_seqno = local_jnl_seqno;
rollback_first = TRUE;
}
if (MAX_SEQNO != resync_seqno)
{
QWASSIGN(*(seq_num *)&reply_msgp->start_seqno[0], resync_seqno);
if (!rollback_first)
{
GRAB_GTMSOURCE_SRV_LATCH(gtmsource_local, max_epoch_interval);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
continue;
srch_status = gtmsource_srch_restart(resync_seqno, recvd_start_flags);
rel_gtmsource_srv_latch(&gtmsource_local->gtmsource_srv_latch);
assert(resync_seqno == gtmsource_local->read_jnl_seqno);
assert(SS_NORMAL == srch_status);
reply_msgp->type = REPL_WILL_RESTART_WITH_INFO;
reply_msgp->jnl_ver = this_side->jnl_ver;
temp_ulong = (0 == this_side->is_std_null_coll) ? START_FLAG_NONE : START_FLAG_COLL_M;
GTMTRIG_ONLY(
assert(this_side->trigger_supported);
temp_ulong |= START_FLAG_TRIGGER_SUPPORT;
)
PUT_ULONG(reply_msgp->start_flags, temp_ulong);
recvd_start_flags = START_FLAG_NONE;
gtmsource_repl_send((repl_msg_ptr_t)reply_msgp, "REPL_WILL_RESTART_WITH_INFO",
resync_seqno, INVALID_SUPPL_STRM);
} else
{ /* Secondary needs to first do FETCHRESYNC rollback to synchronize with primary */
reply_msgp->type = REPL_ROLLBACK_FIRST;
gtmsource_repl_send((repl_msg_ptr_t)reply_msgp, "REPL_ROLLBACK_FIRST",
resync_seqno, INVALID_SUPPL_STRM);
}
}
} else
{ /* REPL_FETCH_RESYNC received and state is WAITING_FOR_RESTART */
if (rollback_first || secondary_ahead)
{ /* Primary and Secondary are currently not in sync */
if (!rollback_first)
{ /* We know the secondary is ahead of the primary in terms of journal seqno but the last
* histinfo records are identical. This means that the secondary is in sync with the
* primary until the primary's journal seqno ("local_jnl_seqno") which should be the new
* resync seqno.
*/
resync_seqno = local_jnl_seqno;
} else
{ /* Determine the resync seqno between this primary and secondary by comparing
* local and remote histinfo records from the tail of the instance file until we reach
* one seqno whose histinfo information is identical in both.
*/
assert(1 != recvd_seqno);
resync_seqno = gtmsource_find_resync_seqno(&local_histinfo, &remote_histinfo);
assert((MAX_SEQNO != resync_seqno) || (GTMSOURCE_CHANGING_MODE == gtmsource_state)
|| (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
|| (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state));
}
} else
{ /* Primary and Secondary are in sync upto "recvd_seqno". Send it back as the new resync seqno. */
resync_seqno = recvd_seqno;
}
if (MAX_SEQNO != resync_seqno)
{
assert(GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state && REPL_FETCH_RESYNC == recvd_msg_type);
reply_msgp->type = REPL_RESYNC_SEQNO;
QWASSIGN(*(seq_num *)&reply_msgp->start_seqno[0], resync_seqno);
gtmsource_repl_send((repl_msg_ptr_t)reply_msgp, "REPL_RESYNC_SEQNO",
resync_seqno, INVALID_SUPPL_STRM);
}
}
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
return (SS_NORMAL); /* "gtmsource_repl_send" or "gtmsource_find_resync_seqno" did not complete */
if ((GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) || (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state))
continue; /* "gtmsource_repl_send" or "gtmsource_find_resync_seqno" did not complete */
assert(MAX_SEQNO != resync_seqno);
/* After having established connection, initialize a few fields in the gtmsource_local
* structure and flush those changes to the instance file on disk.
*/
GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
continue;
gtmsource_local->connect_jnl_seqno = jctl->jnl_seqno;
gtmsource_local->send_losttn_complete = jctl->send_losttn_complete;
/* Now that "connect_jnl_seqno" has been updated, flush it to corresponding gtmsrc_lcl on disk */
repl_inst_flush_gtmsrc_lcl(); /* this requires the jnlpool lock to be held */
rel_lock(jnlpool.jnlpool_dummy_reg);
if (REPL_WILL_RESTART_WITH_INFO != reply_msgp->type)
{
assert(reply_msgp->type == REPL_RESYNC_SEQNO || reply_msgp->type == REPL_ROLLBACK_FIRST);
if ((REPL_RESYNC_SEQNO == reply_msgp->type) && secondary_was_rootprimary)
{
repl_log(gtmsource_log_fp, TRUE, TRUE, "Sent REPL_RESYNC_SEQNO message with SEQNO %llu [0x%llx]\n",
(*(seq_num *)&reply_msgp->start_seqno[0]), (*(seq_num *)&reply_msgp->start_seqno[0]));
region_top = gd_header->regions + gd_header->n_regions;
assert(NULL != jnlpool.jnlpool_dummy_reg);
GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
continue;
zqgblmod_seqno = jctl->max_zqgblmod_seqno;
if (0 == zqgblmod_seqno)
{ /* If zqgblmod_seqno in all file headers is 0, it implies that this is the first
* FETCHRESYNC rollback after the most recent MUPIP REPLIC -LOSTTNCOMPLETE command.
* Therefore reset zqgblmod_seqno to the rollback seqno. If not the first rollback,
* then zqgblmod_seqno will be reset only if the new rollback seqno is lesser
* than the current value.
*/
zqgblmod_seqno = MAX_SEQNO; /* actually 0xFFFFFFFFFFFFFFFF (max possible seqno) */
/* Reset any pending MUPIP REPLIC -SOURCE -LOSTTNCOMPLETE */
jctl->send_losttn_complete = FALSE;
gtmsource_local->send_losttn_complete = jctl->send_losttn_complete;
}
rel_lock(jnlpool.jnlpool_dummy_reg);
REPL_DPRINT2("BEFORE FINDING RESYNC - zqgblmod_seqno is %llu", zqgblmod_seqno);
REPL_DPRINT2(", curr_seqno is %llu\n", jctl->jnl_seqno);
if (zqgblmod_seqno > resync_seqno)
{ /* reset "zqgblmod_seqno" and "zqgblmod_tn" in all fileheaders to "resync_seqno" */
if (SS_NORMAL != gtmsource_update_zqgblmod_seqno_and_tn(resync_seqno))
{
assert(GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
continue;
}
}
}
/* Could send a REPL_CLOSE_CONN message here */
/* It is expected that on receiving this msg, the Receiver Server will break the connection and exit. */
repl_close(&gtmsource_sock_fd);
LONG_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_TO_QUIT); /* may not be needed after REPL_CLOSE_CONN is sent */
gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION;
continue;
}
/* Now that REPL_WILL_RESTART_WITH_INFO message has been sent, if compression of the replication stream is
* requested, check if the receiver server supports ability to decompress. Dont do this if this receiver has
* previously sent a REPL_CMP2UNCMP message.
*/
gtmsource_local->repl_zlib_cmp_level = repl_zlib_cmp_level = ZLIB_CMPLVL_NONE; /* no compression by default */
if (!gtmsource_received_cmp2uncmp_msg && (ZLIB_CMPLVL_NONE != gtm_zlib_cmp_level))
{
if (REPL_PROTO_VER_MULTISITE_CMP <= remote_side->proto_ver)
{ /* Receiver server is running a version of GT.M that supports compression of replication stream.
* Send test message with compressed data to check if it is able to decompress properly. If so,
* enable compression on the replication pipe. Compression level set in repl_zlib_cmp_level.
*/
if (!gtmsource_get_cmp_info(&repl_zlib_cmp_level))
{
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
return (SS_NORMAL);
else if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
continue;
else
{ /* Got a REPL_XOFF_ACK_ME from receiver. Restart the initial handshake */
assert(GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state);
continue;
}
}
/* Note down replication cmp_level in this source-server specific structure in journal pool */
gtmsource_local->repl_zlib_cmp_level = repl_zlib_cmp_level;
} else
{
repl_log(gtmsource_log_fp, TRUE, FALSE,
"Receiver server does not support compressed data on the replication pipe\n");
repl_log(gtmsource_log_fp, TRUE, FALSE, "Defaulting to NO compression\n");
}
}
if (QWLT(gtmsource_local->read_jnl_seqno, sav_read_jnl_seqno) && (NULL != repl_ctl_list))
{ /* The journal files may have been positioned ahead of the read_jnl_seqno for the next read.
* Indicate that they have to be repositioned into the past.
*/
assert(READ_FILE == gtmsource_local->read_state);
gtmsource_set_lookback();
}
poll_time = poll_immediate;
gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_SENDING_JNLRECS;
assert(1 <= gtmsource_local->read_jnl_seqno);
/* Now that "gtmsource_local->read_jnl_seqno" is initialized, ensure the first send gets logged. */
gtmsource_reinit_logseqno();
gtmsource_init_heartbeat();
/* Internal filters are needed as long as the filter format of the originating side is greater or equal to the
* filter format of the secondary side
*/
if ((this_side->jnl_ver >= remote_side->jnl_ver)
&& (IF_NONE != repl_filter_cur2old[remote_side->jnl_ver - JNL_VER_EARLIEST_REPL]))
{
assert(IF_INVALID != repl_filter_cur2old[remote_side->jnl_ver - JNL_VER_EARLIEST_REPL]);
assert(IF_INVALID != repl_filter_old2cur[remote_side->jnl_ver - JNL_VER_EARLIEST_REPL]);
/* reverse transformation should exist */
assert(IF_NONE != repl_filter_old2cur[remote_side->jnl_ver - JNL_VER_EARLIEST_REPL]);
if (this_side->is_std_null_coll != remote_side->is_std_null_coll)
remote_side->null_subs_xform = (this_side->is_std_null_coll ?
STDNULL_TO_GTMNULL_COLL : GTMNULL_TO_STDNULL_COLL);
else
remote_side->null_subs_xform = FALSE;
gtmsource_filter |= INTERNAL_FILTER;
/* Any time the ^#t global format version is bumped, the below assert will trip. This way, anyone who bumps
* the trigger label ensures that the internal filter routines in repl_filter.c are accordingly changed to
* downgrade triggers before sending them across to a receiver that understands ONLY the prior ^#t format.
*/
assert(0 == MEMCMP_LIT(HASHT_GBL_CURLABEL, "2"));
gtmsource_alloc_filter_buff(gtmsource_msgbufsiz);
} else
{
gtmsource_filter &= ~INTERNAL_FILTER;
if (NO_FILTER == gtmsource_filter)
gtmsource_free_filter_buff();
}
catchup = FALSE;
force_recv_check = TRUE;
xon_wait_logged = FALSE;
/* Flush "gtmsource_local->read_jnl_seqno" to disk right now.
* This will serve as a reference point for next timed flush to occur.
*/
gtmsource_flush_fh(gtmsource_local->read_jnl_seqno);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
continue;
gtmsource_local->send_new_histrec = TRUE; /* Send new histinfo unconditionally at start of connection */
gtmsource_local->next_histinfo_seqno = MAX_SEQNO; /* Initial value. Reset by "gtmsource_send_new_histrec" below */
assert(-1 == gtmsource_local->next_histinfo_num);
while (TRUE)
{
gtmsource_poll_actions(TRUE);
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
return (SS_NORMAL);
if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
break;
if (gtmsource_local->send_losttn_complete)
{ /* Send LOSTTNCOMPLETE across to the secondary and reset flag to FALSE */
losttncomplete_msg.type = REPL_LOSTTNCOMPLETE;
losttncomplete_msg.len = MIN_REPL_MSGLEN;
gtmsource_repl_send((repl_msg_ptr_t)&losttncomplete_msg, "REPL_LOSTTNCOMPLETE",
MAX_SEQNO, INVALID_SUPPL_STRM);
GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
break; /* the outerloop will continue */
gtmsource_local->send_losttn_complete = FALSE;
rel_lock(jnlpool.jnlpool_dummy_reg);
}
if (gtmsource_local->send_new_histrec)
{ /* We are at the beginning of a new histinfo record boundary. Send a REPL_HISTREC message
* before sending journal records for seqnos corresponding to this histinfo.
*/
assert(REPL_PROTO_VER_MULTISITE <= remote_side->proto_ver);
/* Remote version supports multi-site functionality. Send REPL_HISTREC and friends */
gtmsource_send_new_histrec();
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
return (SS_NORMAL);
if ((GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
|| (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state))
break;
assert(FALSE == gtmsource_local->send_new_histrec);
}
/* If the backlog is high, we want to avoid communication overhead as much as possible. We switch
* our communication mode to *catchup* mode, wherein we don't wait for the pipe to become ready to
* send. Rather, we assume the pipe is ready for sending. The risk is that the send may block if
* the pipe is not ready for sending. In the user's perspective, the risk is that the source server
* may not respond to administrative actions such as "change log", "shutdown" (although mupip stop
* would work).
*/
pre_read_seqno = gtmsource_local->read_jnl_seqno;
prev_catchup = catchup;
assert(jctl->write_addr >= gtmsource_local->read_addr);
backlog_bytes = jctl->write_addr - gtmsource_local->read_addr;
GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
break; /* the outerloop will continue */
jnl_seqno = jctl->jnl_seqno;
rel_lock(jnlpool.jnlpool_dummy_reg);
assert(jnl_seqno >= pre_read_seqno - 1); /* jnl_seqno >= pre_read_seqno is the most common case;
* see gtmsource_readpool() for when the rare case can occur */
backlog_count = (jnl_seqno >= pre_read_seqno) ? (jnl_seqno - pre_read_seqno) : 0;
catchup = (BACKLOG_BYTES_THRESHOLD <= backlog_bytes || BACKLOG_COUNT_THRESHOLD <= backlog_count);
if (!prev_catchup && catchup) /* transition from non catchup to catchup */
{
repl_log(gtmsource_log_fp, TRUE, TRUE, "Source server entering catchup mode at Seqno %llu "
"[0x%llx] : Current backlog %llu [0x%llx] : Backlog size in journal pool %llu [0x%llx] "
"bytes\n", pre_read_seqno, pre_read_seqno, backlog_count, backlog_count,
backlog_bytes, backlog_bytes);
prev_now = gtmsource_now;
prev_msg_sent = repl_source_msg_sent;
force_recv_check = TRUE;
} else if (prev_catchup && !catchup) /* transition from catchup to non catchup */
{
repl_log(gtmsource_log_fp, TRUE, TRUE, "Source server returning to regular mode from catchup mode "
"at Seqno %llu [0x%llx] : Current backlog %llu [0x%llx] : Backlog size in journal pool "
" %llu [0x%llx] bytes\n", pre_read_seqno, pre_read_seqno, backlog_count, backlog_count,
backlog_bytes, backlog_bytes);
if (gtmsource_msgbufsiz - MAX_REPL_MSGLEN > 2 * OS_PAGE_SIZE)
{/* We have expanded the buffer by too much (could have been avoided had we sent one transaction
* at a time while reading from journal files); let's revert back to our initial buffer size.
* If we don't reduce our buffer, it is possible that the buffer keeps growing (while reading
* from journal file) thus making the size of sends while reading from journal pool very
* large (> 1 MB). Better to do some house keeping. We will force an expansion if the transaction
* size dictates it. Ideally, this must be done while switching reading back from files to
* pool, but we can't afford to free the buffer until we sent the transaction out. That apart,
* let's wait for some breathing time to do house keeping. In catchup mode, we intend to keep
* the send size large. */
gtmsource_free_filter_buff();
gtmsource_free_msgbuff();
gtmsource_alloc_msgbuff(MAX_REPL_MSGLEN); /* will also allocate filter buffer */
}
}
/* Check if receiver sent us any control message.
* Typically, the traffic from receiver to source is very low compared to traffic in the other direction.
* More often than not, there will be nothing on the pipe to receive. Ideally, we should let TCP
* notify us when there is data on the pipe (async I/O on Unix and VMS). We are not there yet. Until then,
* we use heuristics - attempt receive every GTMSOURCE_SENT_THRESHOLD_FOR_RECV bytes of sent data, and
* every heartbeat period, OR whenever we want to force a check
*/
if ((GTMSOURCE_SENDING_JNLRECS != gtmsource_state) || !catchup || prev_now != (save_now = gtmsource_now)
|| (GTMSOURCE_SENT_THRESHOLD_FOR_RECV <= (repl_source_msg_sent - prev_msg_sent))
|| force_recv_check)
{
REPL_EXTRA_DPRINT2("gtmsource_process: receiving because : %s\n",
(GTMSOURCE_SENDING_JNLRECS != gtmsource_state) ? "state is not SENDING_JNLRECS" :
!catchup ? "not in catchup mode" :
(prev_now != save_now) ? "heartbeat interval passed" :
(GTMSOURCE_SENT_THRESHOLD_FOR_RECV <= repl_source_msg_sent - prev_msg_sent) ?
"sent bytes threshold for recv crossed" :
"force recv check");
REPL_EXTRA_DPRINT6("gtmsource_state : %d prev_now : %ld gtmsource_now : %ld repl_source_msg_sent "
": %ld prev_msg_sent : %ld\n", gtmsource_state, prev_now, save_now,
repl_source_msg_sent, prev_msg_sent);
REPL_RECV_LOOP(gtmsource_sock_fd, gtmsource_msgp, MIN_REPL_MSGLEN, FALSE, &poll_time)
{
if (0 == recvd_len) /* nothing received in the first attempt, let's try again later */
break;
gtmsource_poll_actions(TRUE);
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
return (SS_NORMAL);
if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
break;
}
REPL_EXTRA_DPRINT3("gtmsource_process: %d received, type is %d\n", recvd_len,
(0 != recvd_len) ? gtmsource_msgp->type : -1);
if (GTMSOURCE_SENDING_JNLRECS == gtmsource_state && catchup)
{
if (prev_now != save_now)
prev_now = save_now;
else if (GTMSOURCE_SENT_THRESHOLD_FOR_RECV <= repl_source_msg_sent - prev_msg_sent)
{ /* do not set to repl_source_msg_sent; increment by GTMSOURCE_SENT_THRESHOLD_FOR_RECV
* instead so that we force recv every GTMSOURCE_SENT_THRESHOLD_FOR_RECV bytes */
prev_msg_sent += GTMSOURCE_SENT_THRESHOLD_FOR_RECV;
}
}
} else
{ /* behave as if there was nothing to be read */
status = SS_NORMAL;
recvd_len = 0;
}
force_recv_check = FALSE;
if (SS_NORMAL == status && 0 != recvd_len)
{ /* Process the received control message */
assert(MIN_REPL_MSGLEN == recvd_len);
/* One is not always guaranteed the received message is in source native endian format.
* See endianness related comments in gtmsource_recv_restart for why. So be safe and handle
* it just like how gtmsource_recv_restart does. The below check works as all messages we
* expect at this point have a fixed length of MIN_REPL_MSGLEN.
*/
msg_is_cross_endian = (((unsigned)MIN_REPL_MSGLEN < (unsigned)gtmsource_msgp->len)
&& ((unsigned)MIN_REPL_MSGLEN == GTM_BYTESWAP_32((unsigned)gtmsource_msgp->len)));
if (msg_is_cross_endian)
{
gtmsource_msgp->type = GTM_BYTESWAP_32(gtmsource_msgp->type);
gtmsource_msgp->len = GTM_BYTESWAP_32(gtmsource_msgp->len);
}
assert(MIN_REPL_MSGLEN == gtmsource_msgp->len);
assert(remote_side->endianness_known);
/* Even though we know the endianness of the remote side at this point, we use
* msg_is_cross_endian as it should be a safer alternative in case of messages
* like REPL_XOFF_ACK_ME where the receiver (particularly pre-V55000 versions)
* is not careful to send in a consistent endian format across versions. So we
* use the endianness of this particular message rather than the endianness of
* the connection.
*/
switch(gtmsource_msgp->type)
{
case REPL_XOFF:
case REPL_XOFF_ACK_ME:
gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_XON;
poll_time = poll_wait;
repl_log(gtmsource_log_fp, TRUE, TRUE,
"REPL_XOFF/REPL_XOFF_ACK_ME received. Send stalled...\n");
xon_wait_logged = FALSE;
if (REPL_XOFF_ACK_ME == gtmsource_msgp->type)
{
xoff_ack.type = REPL_XOFF_ACK;
tmp_seqno = *(seq_num *)&gtmsource_msgp->msg[0];
if (msg_is_cross_endian)
tmp_seqno = GTM_BYTESWAP_64(tmp_seqno);
*(seq_num *)&xoff_ack.msg[0] = tmp_seqno;
xoff_ack.len = MIN_REPL_MSGLEN;
gtmsource_repl_send((repl_msg_ptr_t)&xoff_ack, "REPL_XOFF_ACK",
MAX_SEQNO, INVALID_SUPPL_STRM);
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
return (SS_NORMAL); /* "gtmsource_repl_send" did not complete */
if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
break; /* "gtmsource_repl_send" did not complete */
}
break;
case REPL_XON:
gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_SENDING_JNLRECS;
poll_time = poll_immediate;
repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL_XON received\n");
gtmsource_restart_heartbeat; /*Macro*/
REPL_DPRINT1("Restarting HEARTBEAT\n");
xon_wait_logged = FALSE;
/* In catchup mode, we do not receive as often as we would have in non catchup mode.
* The consequence of this may be that we do not react to XOFF quickly enough,
* making it worse for the replication pipe. We may end up with multiple XOFFs (with
* intervening XONs) sitting on the replication pipe that are yet to be received
* and processed by the source server. To avoid such a situation, on receipt of
* an XON, we immediately force a check on the incoming pipe thereby draining
* all pending XOFF/XONs, keeping the pipe smooth. Also, there is less likelihood
* of missing a HEARTBEAT response (that is perhaps on the pipe) which may lead to
* connection breakage although the pipe is alive and well.
*
* We will force a check regardless of mode (catchup or non catchup) as we may
* be pounding the secondary even in non catchup mode
*/
force_recv_check = TRUE;
break;
case REPL_BADTRANS:
case REPL_CMP2UNCMP:
case REPL_START_JNL_SEQNO:
QWASSIGN(recvd_seqno, *(seq_num *)&gtmsource_msgp->msg[0]);
if (msg_is_cross_endian)
recvd_seqno = GTM_BYTESWAP_64(recvd_seqno);
gtmsource_state = gtmsource_local->gtmsource_state
= GTMSOURCE_SEARCHING_FOR_RESTART;
if ((REPL_BADTRANS == gtmsource_msgp->type)
|| (REPL_CMP2UNCMP == gtmsource_msgp->type))
{
already_communicated = TRUE;
recvd_start_flags = START_FLAG_NONE;
if (REPL_BADTRANS == gtmsource_msgp->type)
repl_log(gtmsource_log_fp, TRUE, TRUE, "Received REPL_BADTRANS "
"message with SEQNO %llu [0x%llx]\n",
recvd_seqno, recvd_seqno);
else
{
repl_log(gtmsource_log_fp, TRUE, TRUE, "Received REPL_CMP2UNCMP "
"message with SEQNO %llu [0x%llx]\n",
recvd_seqno, recvd_seqno);
repl_log(gtmsource_log_fp, TRUE, FALSE,
"Defaulting to NO compression for this connection\n");
gtmsource_received_cmp2uncmp_msg = TRUE;
}
} else
{
recvd_start_flags = ((repl_start_msg_ptr_t)gtmsource_msgp)->start_flags;
if (msg_is_cross_endian)
recvd_start_flags = GTM_BYTESWAP_32(recvd_start_flags);
already_communicated = FALSE;
repl_log(gtmsource_log_fp, TRUE, TRUE,
"Received REPL_START_JNL_SEQNO message with SEQNO %llu [0x%llx]. "
"Possible crash of recvr/update process\n",
recvd_seqno, recvd_seqno);
}
break;
case REPL_HEARTBEAT:
if (msg_is_cross_endian)
{
heartbeat_msg = (repl_heartbeat_msg_ptr_t)gtmsource_msgp;
tmp_seqno = *(seq_num *)&heartbeat_msg->ack_seqno[0];
tmp_seqno = GTM_BYTESWAP_64(tmp_seqno);
*(seq_num *)&heartbeat_msg->ack_seqno[0] = tmp_seqno;
tmp_time4 = *(gtm_time4_t *)&heartbeat_msg->ack_time[0];
tmp_time4 = GTM_BYTESWAP_32(tmp_time4);
*(gtm_time4_t *)&heartbeat_msg->ack_time[0] = tmp_time4;
}
gtmsource_process_heartbeat((repl_heartbeat_msg_ptr_t)gtmsource_msgp);
break;
default:
repl_log(gtmsource_log_fp, TRUE, TRUE, "Message of unknown type %d of length %d "
"bytes received; hex dump follows\n", gtmsource_msgp->type, recvd_len);
for (index = 0; index < MIN(recvd_len, gtmsource_msgbufsiz - REPL_MSG_HDRLEN); )
{
repl_log(gtmsource_log_fp, FALSE, FALSE, "%.2x ",
gtmsource_msgp->msg[index]);
if ((++index) % MAX_HEXDUMP_CHARS_PER_LINE == 0)
repl_log(gtmsource_log_fp, FALSE, TRUE, "\n");
}
repl_log(gtmsource_log_fp, FALSE, TRUE, "\n"); /* flush BEFORE the assert */
assert(FALSE);
break;
}
} else if (SS_NORMAL != status)
{
if (EREPL_RECV == repl_errno)
{
if (REPL_CONN_RESET(status))
{
/* Connection reset */
repl_log(gtmsource_log_fp, TRUE, TRUE,
"Connection reset while attempting to receive from secondary."
" Status = %d ; %s\n", status, STRERROR(status));
repl_close(&gtmsource_sock_fd);
SHORT_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_CLOSE_CONN);
gtmsource_state = gtmsource_local->gtmsource_state
= GTMSOURCE_WAITING_FOR_CONNECTION;
break;
} else
{
SNPRINTF(err_string, SIZEOF(err_string),
"Error receiving Control message from Receiver. Error in recv : %s",
STRERROR(status));
rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_STRING(err_string));
}
} else if (EREPL_SELECT == repl_errno)
{
SNPRINTF(err_string, SIZEOF(err_string),
"Error receiving Control message from Receiver. Error in select : %s",
STRERROR(status));
rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_STRING(err_string));
}
}
if (GTMSOURCE_WAITING_FOR_XON == gtmsource_state)
{
if (!xon_wait_logged)
{
repl_log(gtmsource_log_fp, TRUE, TRUE, "Waiting to receive XON\n");
gtmsource_stall_heartbeat; /* Macro */
REPL_DPRINT1("Stalling HEARTBEAT\n");
xon_wait_logged = TRUE;
}
if (GTMSOURCE_FH_FLUSH_INTERVAL <= difftime(gtmsource_now, gtmsource_last_flush_time))
{
gtmsource_flush_fh(gtmsource_local->read_jnl_seqno);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
break; /* the outerloop will continue */
}
continue;
}
if (GTMSOURCE_SEARCHING_FOR_RESTART == gtmsource_state ||
GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
{
xon_wait_logged = FALSE;
break;
}
assert(gtmsource_state == GTMSOURCE_SENDING_JNLRECS);
if (force_recv_check) /* we want to poll the incoming pipe for possible XOFF */
continue;
assert(pre_read_seqno == gtmsource_local->read_jnl_seqno);
GRAB_GTMSOURCE_SRV_LATCH(gtmsource_local, max_epoch_interval);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
break; /* the outerloop will continue */
tot_tr_len = gtmsource_get_jnlrecs(&gtmsource_msgp->msg[0], &data_len,
gtmsource_msgbufsiz - REPL_MSG_HDRLEN,
!(gtmsource_filter & EXTERNAL_FILTER));
rel_gtmsource_srv_latch(&gtmsource_local->gtmsource_srv_latch);
/* It is safe to send the journal records as we are guaranteed NO online rollback happened in between
* and so we won't be sending garbage
*/
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
return (SS_NORMAL);
if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state)
break;
if (GTMSOURCE_SEND_NEW_HISTINFO == gtmsource_state)
{ /* This is a signal from "gtmsource_get_jnlrecs" to send a REPL_HISTREC message first
* before sending any more seqnos across. Set "gtmsource_local->send_new_histrec" to TRUE.
*/
assert(0 == tot_tr_len);
gtmsource_local->send_new_histrec = TRUE; /* Will cause a new histinfo record to be sent first */
gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_SENDING_JNLRECS;
continue; /* Send a REPL_HISTREC message first and then send journal records */
}
post_read_seqno = gtmsource_local->read_jnl_seqno;
if (0 <= tot_tr_len)
{
if (0 < data_len)
{
APPLY_EXT_FILTER_IF_NEEDED(gtmsource_filter, gtmsource_msgp, data_len, tot_tr_len);
gtmsource_msgp->type = REPL_TR_JNL_RECS;
gtmsource_msgp->len = data_len + REPL_MSG_HDRLEN;
send_msgp = gtmsource_msgp;
send_tr_len = tot_tr_len;
intfilter_error = FALSE;
if (gtmsource_filter & INTERNAL_FILTER)
{
in_buff = gtmsource_msgp->msg;
in_buflen = data_len; /* size of the first journal record in the converted buffer */
out_buffmsg = repl_filter_buff;
out_buff = out_buffmsg + REPL_MSG_HDRLEN;
out_bufsiz = repl_filter_bufsiz - REPL_MSG_HDRLEN;
remaining_len = tot_tr_len;
while (JREC_PREFIX_SIZE <= remaining_len)
{
filter_seqno = ((struct_jrec_null *)(in_buff))->jnl_seqno;
DEBUG_ONLY(
save_inbuff = in_buff;
save_outbuff = out_buff;
)
APPLY_INT_FILTER(in_buff, in_buflen, out_buff, out_buflen,
out_bufsiz, status);
/* Internal filters should not modify the incoming pointers. Assert that. */
assert((save_inbuff == in_buff) && (save_outbuff == out_buff));
if (SS_NORMAL == status)
{ /* adjust various pointers and book-keeping values to move to next
* record.
*/
((repl_msg_ptr_t)(out_buffmsg))->type = REPL_TR_JNL_RECS;
((repl_msg_ptr_t)(out_buffmsg))->len = out_buflen + REPL_MSG_HDRLEN;
out_buffmsg = (out_buff + out_buflen);
remaining_len -= (in_buflen + REPL_MSG_HDRLEN);
assert(0 <= remaining_len);
if (0 >= remaining_len)
break;
in_buff += in_buflen;
in_buflen = ((repl_msg_ptr_t)(in_buff))->len - REPL_MSG_HDRLEN;
in_buff += REPL_MSG_HDRLEN;
out_buff = (out_buffmsg + REPL_MSG_HDRLEN);
out_bufsiz -= (out_buflen + REPL_MSG_HDRLEN);
assert(0 <= (int)out_bufsiz);
} else if (EREPL_INTLFILTER_NOSPC == repl_errno)
{
REALLOCATE_INT_FILTER_BUFF(out_buff, out_buffmsg, out_bufsiz);
/* note that in_buff and in_buflen is not changed so that we can
* start from where we left
*/
} else /* fatal error from the internal filter */
{
intfilter_error = TRUE;
break;
}
}
assert((0 == remaining_len) || intfilter_error);
GTMTRIG_ONLY(ISSUE_TRIG2NOTRIG_IF_NEEDED;)
send_msgp = (repl_msg_ptr_t)repl_filter_buff;
send_tr_len = out_buffmsg - repl_filter_buff;
if (0 == send_tr_len)
{ /* This is possible ONLY if the first transaction in the buffer read from
* journal pool or disk encountered error while doing internal filter
* conversion. Issue rts_error right away as there is nothing much we can
* do at this point.
*/
assert(intfilter_error);
assert((EREPL_INTLFILTER_BADREC == repl_errno)
|| (EREPL_INTLFILTER_REPLGBL2LONG == repl_errno)
|| (EREPL_INTLFILTER_SECNODZTRIGINTP == repl_errno)
|| (EREPL_INTLFILTER_MULTILINEXECUTE == repl_errno));
assert(filter_seqno == pre_read_seqno);
INT_FILTER_RTS_ERROR(filter_seqno); /* no return */
}
}
assert(send_tr_len && (0 == (send_tr_len % REPL_MSG_ALIGN)));
/* ensure that the head of the buffer has the correct type and len */
assert((REPL_TR_JNL_RECS == send_msgp->type)
&& (0 == (send_msgp->len % JNL_REC_START_BNDRY)));
/* At this point send_msgp is the buffer to be sent and send_tr_len is the send size */
assert(remote_side->endianness_known);
if (remote_side->cross_endian && (this_side->jnl_ver < remote_side->jnl_ver))
{ /* Cross-endian replication with GT.M version on primary being lesser than that
* the secondary. Do the endian conversion in the primary so that the secondary
* can consume it as-is.
* No return if the below call to repl_tr_endian_convert fails.
*/
repl_tr_endian_convert(send_msgp, send_tr_len, pre_read_seqno);
}
pre_cmpmsglen = send_tr_len; /* send_tr_len will be updated below */
if (ZLIB_CMPLVL_NONE != repl_zlib_cmp_level)
{ /* Compress the journal records before replicating them across the pipe.
* Depending on whether the total data length to be sent is within a threshold
* or not (see repl_msg.h before REPL_TR_CMP_THRESHOLD #define for why), send
* either a REPL_TR_CMP_JNL_RECS or REPL_TR_CMP_JNL_RECS2 message
*/
msghdrlen = (REPL_TR_CMP_THRESHOLD > send_tr_len)
? REPL_MSG_HDRLEN : REPL_MSG_HDRLEN2;
cmpbuflen = gtmsource_cmpmsgbufsiz - msghdrlen;
cmpbufptr = ((Bytef *)gtmsource_cmpmsgp) + msghdrlen;
ZLIB_COMPRESS(cmpbufptr, cmpbuflen, send_msgp, send_tr_len,
repl_zlib_cmp_level, cmpret);
BREAK_IF_CMP_ERROR(cmpret, send_tr_len); /* Note: break stmt. inside the macro */
if (Z_OK == cmpret)
{ /* Send compressed buffer */
send_msgp = gtmsource_cmpmsgp;
if (REPL_TR_CMP_THRESHOLD > send_tr_len)
{ /* Send REPL_TR_CMP_JNL_RECS message with 8-byte header */
SET_8BYTE_CMP_MSGHDR(send_msgp, send_tr_len, cmpbuflen, msghdrlen);
} else
{ /* Send REPL_TR_CMP_JNL_RECS2 message with 16-byte header */
SET_16BYTE_CMP_MSGHDR(send_msgp, send_tr_len, cmpbuflen, msghdrlen);
}
} else
{ /* Send normal buffer */
repl_log(gtmsource_log_fp, TRUE, FALSE, "Defaulting to NO compression\n");
repl_zlib_cmp_level = ZLIB_CMPLVL_NONE; /* no compression */
gtmsource_local->repl_zlib_cmp_level = repl_zlib_cmp_level;
}
}
assert((send_tr_len == pre_cmpmsglen) || (ZLIB_CMPLVL_NONE != repl_zlib_cmp_level));
assert(0 == (send_tr_len % REPL_MSG_ALIGN));
/* The following loop tries to send multiple seqnos in one shot. resync_seqno gets
* updated once the send is completely successful. If an error occurs in the middle
* of the send, it is possible that we successfully sent a few seqnos to the other side.
* In this case resync_seqno should be updated to reflect those seqnos. Not doing so
* might cause the secondary to get ahead of the primary in terms of resync_seqno.
* Although it is possible to determine the exact seqno where the send partially failed,
* we update resync_seqno as if all seqnos were successfully sent (It is ok for the
* resync_seqno on the primary side to be a little more than the actual value as long as
* the secondary side has an accurate value of resync_seqno. This is because the
* resync_seqno of the system is the minimum of the resync_seqno of both primary
* and secondary). This is done by the call to gtmsource_flush_fh() done within the
* REPL_SEND_LOOP macro as well as in the (SS_NORMAL != status) if condition below.
* Note that all of this is applicable only in a dualsite replication scenario. In
* case of a multisite scenario, it is always the receiver server that tells the
* sequence number from where the source server should start sending. So, even if
* the source server notes down a higher value of journal sequence number in
* jnlpool.gtmsource_local->read_jnl_seqno, it is not a problem since the receiver
* server will communicate the appropriate sequence number as part of the histinfo
* exchange.
*/
REPL_SEND_LOOP(gtmsource_sock_fd, send_msgp, send_tr_len, catchup, &poll_immediate)
{
gtmsource_poll_actions(FALSE);
if (GTMSOURCE_CHANGING_MODE == gtmsource_state)
{
gtmsource_flush_fh(post_read_seqno);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
break;
return (SS_NORMAL);
}
}
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
break; /* the outerloop will continue */
if (SS_NORMAL != status)
{
gtmsource_flush_fh(post_read_seqno);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
break; /* the outerloop will continue */
if (REPL_CONN_RESET(status) && EREPL_SEND == repl_errno)
{
repl_log(gtmsource_log_fp, TRUE, TRUE,
"Connection reset while sending seqno data from "
"%llu [0x%llx] to %llu [0x%llx]. Status = %d ; %s\n",
pre_read_seqno, pre_read_seqno, post_read_seqno, post_read_seqno,
status, STRERROR(status));
repl_close(&gtmsource_sock_fd);
SHORT_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_CLOSE_CONN);
gtmsource_state = gtmsource_local->gtmsource_state
= GTMSOURCE_WAITING_FOR_CONNECTION;
break;
}
if (EREPL_SEND == repl_errno)
{
SNPRINTF(err_string, SIZEOF(err_string),
"Error sending DATA. Error in send : %s", STRERROR(status));
rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2,
RTS_ERROR_STRING(err_string));
}
if (EREPL_SELECT == repl_errno)
{
SNPRINTF(err_string, SIZEOF(err_string),
"Error sending DATA. Error in select : %s", STRERROR(status));
rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2,
RTS_ERROR_STRING(err_string));
}
}
if (intfilter_error)
{ /* Now that we are done sending whatever buffer was filter converted, issue
* the error. This will bring down the source server (due to the rts_error).
* At this point, jnlpool.gtmsource_local->read_jnl_seqno could effectively
* be behind the receiver server's journal sequence number. But, that is
* okay since as part of reconnection (when the source server comes back up),
* the receiever server will communicate the appropriate sequence number as
* part of the histinfo exchange.
*/
assert((EREPL_INTLFILTER_BADREC == repl_errno)
|| (EREPL_INTLFILTER_REPLGBL2LONG == repl_errno)
|| (EREPL_INTLFILTER_SECNODZTRIGINTP == repl_errno)
|| (EREPL_INTLFILTER_MULTILINEXECUTE == repl_errno));
assert(filter_seqno <= post_read_seqno);
INT_FILTER_RTS_ERROR(filter_seqno); /* no return */
}
jnlpool.gtmsource_local->read_jnl_seqno = post_read_seqno;
if (GTMSOURCE_FH_FLUSH_INTERVAL <= difftime(gtmsource_now, gtmsource_last_flush_time))
{
gtmsource_flush_fh(post_read_seqno);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
break; /* the outerloop will continue */
}
repl_source_cmp_sent += (qw_num)send_tr_len;
repl_source_msg_sent += (qw_num)pre_cmpmsglen;
repl_source_data_sent += (qw_num)(pre_cmpmsglen)
- (post_read_seqno - pre_read_seqno) * REPL_MSG_HDRLEN;
log_seqno = post_read_seqno - 1; /* post_read_seqno is the "next" seqno to be sent,
* not the last one we sent */
if (gtmsource_logstats || (log_seqno - lastlog_seqno >= log_interval))
{ /* print always when STATSLOG is ON, or when the log interval has passed */
trans_sent_cnt += (log_seqno - lastlog_seqno);
/* jctl->jnl_seqno >= post_read_seqno is the most common case;
* see gtmsource_readpool() for when the rare case can occur */
jnl_seqno = jctl->jnl_seqno;
assert(jnl_seqno >= post_read_seqno - 1);
diff_seqno = (jnl_seqno >= post_read_seqno) ?
(jnl_seqno - post_read_seqno) : 0;
repl_log(gtmsource_log_fp, FALSE, FALSE, "REPL INFO - Seqno : %llu [0x%llx]",
log_seqno, log_seqno);
repl_log(gtmsource_log_fp, FALSE, FALSE, " Jnl Total : %llu [0x%llx] Msg Total :"
" %llu [0x%llx] CmpMsg Total : %llu [0x%llx] ",
repl_source_data_sent, repl_source_data_sent,
repl_source_msg_sent, repl_source_msg_sent,
repl_source_cmp_sent, repl_source_cmp_sent);
repl_log(gtmsource_log_fp, FALSE, TRUE, "Current backlog : %llu [0x%llx]\n",
diff_seqno, diff_seqno);
/* gtmsource_now is updated by the heartbeat protocol every heartbeat
* interval. To cut down on calls to time(), we use gtmsource_now as the
* time to figure out if we have to log statistics. This works well as the
* logging interval generally is larger than the heartbeat interval, and that
* the heartbeat protocol is running when we are sending data. The consequence
* although is that we may defer logging when we would have logged. We can live
* with that given the benefit of not calling time related system calls.
* Currently, the logging interval is not changeable by users. When/if we provide
* means of choosing log interval, this code may have to be re-examined.
* Vinaya 2003, Sep 08
*/
assert(0 != gtmsource_now); /* must hold if we are sending data */
repl_source_this_log_time = gtmsource_now; /* approximate time, in the worst case,
* behind by heartbeat interval */
assert(repl_source_this_log_time >= repl_source_prev_log_time);
time_elapsed = difftime(repl_source_this_log_time,
repl_source_prev_log_time);
if ((double)GTMSOURCE_LOGSTATS_INTERVAL <= time_elapsed)
{
delta_sent_cnt = trans_sent_cnt - last_log_tr_sent_cnt;
delta_data_sent = repl_source_data_sent
- repl_source_lastlog_data_sent;
delta_msg_sent = repl_source_msg_sent
- repl_source_lastlog_msg_sent;
repl_log(gtmsource_log_fp, TRUE, FALSE, "REPL INFO since last log : "
"Time elapsed : %00.f Tr sent : %llu [0x%llx] Tr bytes : %llu "
"[0x%llx] Msg bytes : %llu [0x%llx]\n", time_elapsed,
delta_sent_cnt, delta_sent_cnt, delta_data_sent, delta_data_sent,
delta_msg_sent, delta_msg_sent);
repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL INFO since last log : "
"Time elapsed : %00.f Tr sent/s : %f Tr bytes/s : %f "
"Msg bytes/s : %f\n", time_elapsed,
(float)delta_sent_cnt / time_elapsed,
(float)delta_data_sent / time_elapsed,
(float)delta_msg_sent / time_elapsed);
repl_source_lastlog_data_sent = repl_source_data_sent;
repl_source_lastlog_msg_sent = repl_source_msg_sent;
last_log_tr_sent_cnt = trans_sent_cnt;
repl_source_prev_log_time = repl_source_this_log_time;
}
lastlog_seqno = log_seqno;
}
poll_time = poll_immediate;
} else /* data_len == 0 */
{ /* nothing to send */
gtmsource_flush_fh(post_read_seqno);
if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)
break; /* the outerloop will continue */
poll_time = poll_wait;
rel_quant(); /* give up processor and let other processes run */
}
} else /* else tot_tr_len < 0, error */
{
if (0 < data_len) /* Insufficient buffer space, increase the buffer space */
gtmsource_alloc_msgbuff(data_len + REPL_MSG_HDRLEN);
else
GTMASSERT; /* Major problems */
}
}
}
}
Reference in New Issue View Git Blame Copy Permalink