/**************************************************************** * * * Copyright 2006, 2012 Fidelity Information Services, Inc.* * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * * the license, please stop and do not read further. * * * ****************************************************************/ #if defined(__MVS__) && !defined(_ISOC99_SOURCE) #define _ISOC99_SOURCE #endif #include "mdef.h" #include "gtm_string.h" #include "gtm_stdio.h" #include "gtm_socket.h" #include "gtm_inet.h" #include "gtm_fcntl.h" #include "gtm_unistd.h" #include "gtm_time.h" #include "gtm_stat.h" #include #include #include #ifdef VMS #include /* Required for gtmsource.h */ #endif #include "gdsroot.h" #include "gdsblk.h" #include "gtm_facility.h" #include "fileinfo.h" #include "gdsbt.h" #include "gdsfhead.h" #include "filestruct.h" #include "repl_msg.h" #include "gtmsource.h" #include "repl_comm.h" #include "jnl.h" #include "hashtab_mname.h" /* needed for muprec.h */ #include "hashtab_int4.h" /* needed for muprec.h */ #include "hashtab_int8.h" /* needed for muprec.h */ #include "buddy_list.h" #include "muprec.h" #include "repl_ctl.h" #include "repl_errno.h" #include "repl_dbg.h" #include "iosp.h" #include "gt_timer.h" #include "gtmsource_heartbeat.h" #include "repl_filter.h" #include "repl_log.h" #include "min_max.h" #include "rel_quant.h" #include "copy.h" #include "ftok_sems.h" #include "repl_instance.h" #include "gtmmsg.h" #include "repl_sem.h" #include "have_crit.h" /* needed for ZLIB_COMPRESS */ #include "deferred_signal_handler.h" /* needed for ZLIB_COMPRESS */ #include "gtm_zlib.h" #include "repl_sort_tr_buff.h" #include "replgbl.h" #include "gtmsource_srv_latch.h" #include "gv_trigger_common.h" #define MAX_HEXDUMP_CHARS_PER_LINE 26 /* 2 characters per byte + space, 80 column assumed */ #define BREAK_IF_CMP_ERROR(CMPRET, SEND_TR_LEN) \ { \ switch(CMPRET) \ { \ case Z_MEM_ERROR: \ repl_log(gtmsource_log_fp, TRUE, FALSE, "Out-of-memory error from compress function " \ "while compressing %d bytes\n", SEND_TR_LEN); \ assert(FALSE); \ break; \ case Z_BUF_ERROR: \ repl_log(gtmsource_log_fp, TRUE, FALSE, "Insufficient output buffer error from compress function " \ "while compressing %d bytes\n", SEND_TR_LEN); \ assert(FALSE); \ break; \ case Z_STREAM_ERROR: \ repl_log(gtmsource_log_fp, TRUE, FALSE, "Compression level %d invalid error from compress function " \ "while compressing %d bytes\n", repl_zlib_cmp_level, SEND_TR_LEN); \ assert(FALSE); \ break; \ } \ } #define SET_8BYTE_CMP_MSGHDR(SEND_MSGP, SEND_TR_LEN, CMPBUFLEN, MSGHDRLEN) \ { \ SEND_MSGP->type = (SEND_TR_LEN << REPL_TR_CMP_MSG_TYPE_BITS) | REPL_TR_CMP_JNL_RECS; \ SEND_MSGP->len = (int4)cmpbuflen + msghdrlen; \ /* Note that a compressed message need not be 8-byte aligned even though the input message was. So round it up to \ * the nearest align boundary. The actual message will contain the unaligned length which is what the receiver will \ * receive. But the # of bytes transmitted across will be the aligned length. \ */ \ SEND_TR_LEN = ROUND_UP(SEND_MSGP->len, REPL_MSG_ALIGN); \ } #define SET_16BYTE_CMP_MSGHDR(SEND_MSGP, SEND_TR_LEN, CMPBUFLEN, MSGHDRLEN) \ { \ repl_cmpmsg_ptr_t send_cmpmsgp; \ \ send_cmpmsgp = (repl_cmpmsg_ptr_t)SEND_MSGP; \ assert(&send_cmpmsgp->type == &SEND_MSGP->type); \ assert(&send_cmpmsgp->len == &SEND_MSGP->len); \ send_cmpmsgp->type = REPL_TR_CMP_JNL_RECS2; \ /* Note that a compressed message need not be 8-byte aligned even though the input message was. So round it up to \ * the nearest align boundary. The actual message will contain the unaligned length which is what the receiver will \ * receive. But the # of bytes transmitted across will be the aligned length. \ */ \ send_cmpmsgp->len = (int4)(ROUND_UP(CMPBUFLEN + MSGHDRLEN, REPL_MSG_ALIGN)); \ send_cmpmsgp->uncmplen = SEND_TR_LEN; \ send_cmpmsgp->cmplen = (int4)CMPBUFLEN; \ SEND_TR_LEN = SEND_MSGP->len; \ } #ifdef GTM_TRIGGER #define ISSUE_TRIG2NOTRIG_IF_NEEDED \ { \ DCL_THREADGBL_ACCESS; \ \ SETUP_THREADGBL_ACCESS; \ if (!(TREF(replgbl)).trig_replic_warning_issued && (TREF(replgbl)).trig_replic_suspect_seqno \ && !remote_side->trigger_supported) \ { /* Note: The below repl_log text is copied from TRIG2NOTRIG error message content from merrors.msg. Change \ * to one should be reflected in another \ */ \ repl_log(gtmsource_log_fp, TRUE, TRUE, "Warning: Sending transaction sequence number %d which used " \ "triggers to a replicator that does not support triggers\n", (TREF(replgbl)).trig_replic_suspect_seqno);\ (TREF(replgbl)).trig_replic_warning_issued = TRUE; /* No more warnings till restart */ \ (TREF(replgbl)).trig_replic_suspect_seqno = seq_num_zero; \ } \ } #endif GBLDEF struct timeval gtmsource_poll_wait, gtmsource_poll_immediate; GBLDEF repl_msg_ptr_t gtmsource_msgp = NULL; GBLDEF int gtmsource_msgbufsiz = 0; GBLDEF repl_msg_ptr_t gtmsource_cmpmsgp = NULL; GBLDEF int gtmsource_cmpmsgbufsiz = 0; GBLDEF boolean_t gtmsource_received_cmp2uncmp_msg; GBLDEF qw_num repl_source_data_sent = 0; GBLDEF qw_num repl_source_msg_sent = 0; GBLDEF qw_num repl_source_cmp_sent = 0; GBLDEF qw_num repl_source_lastlog_data_sent = 0; GBLDEF qw_num repl_source_lastlog_msg_sent = 0; GBLDEF time_t repl_source_prev_log_time; GBLDEF time_t repl_source_this_log_time; GBLDEF time_t gtmsource_last_flush_time; GBLREF gtmsource_state_t gtmsource_state; GBLREF uchar_ptr_t repl_filter_buff; GBLREF int repl_filter_bufsiz; GBLREF volatile time_t gtmsource_now; GBLREF int gtmsource_sock_fd; GBLREF jnlpool_addrs jnlpool; GBLREF gd_addr *gd_header; GBLREF sgmnt_addrs *cs_addrs; GBLREF sgmnt_data_ptr_t cs_data; GBLREF gd_region *gv_cur_region; GBLREF repl_ctl_element *repl_ctl_list; GBLREF gtmsource_options_t gtmsource_options; GBLREF int gtmsource_log_fd; GBLREF int gtmsource_statslog_fd; GBLREF FILE *gtmsource_log_fp; GBLREF FILE *gtmsource_statslog_fp; GBLREF boolean_t gtmsource_logstats; GBLREF int gtmsource_filter; GBLREF gd_addr *gd_header; GBLREF seq_num seq_num_zero, seq_num_minus_one, seq_num_one; GBLREF unsigned int jnl_source_datalen, jnl_dest_maxdatalen; GBLREF unsigned char jnl_source_rectype, jnl_dest_maxrectype; GBLREF int repl_max_send_buffsize, repl_max_recv_buffsize; GBLREF seq_num lastlog_seqno; GBLREF uint4 log_interval; GBLREF qw_num trans_sent_cnt, last_log_tr_sent_cnt; GBLREF repl_conn_info_t *this_side, *remote_side; GBLREF int4 strm_index; GBLREF uint4 process_id; GBLREF seq_num gtmsource_save_read_jnl_seqno; error_def(ERR_JNLNEWREC); error_def(ERR_JNLSETDATA2LONG); error_def(ERR_REPLCOMM); error_def(ERR_REPLFTOKSEM); error_def(ERR_REPLGBL2LONG); error_def(ERR_REPLINSTNOHIST); error_def(ERR_REPLNOMULTILINETRG); error_def(ERR_REPLRECFMT); error_def(ERR_REPLXENDIANFAIL); error_def(ERR_SECNODZTRIGINTP); error_def(ERR_TRIG2NOTRIG); error_def(ERR_TEXT); /* Endian converts the given set of journal records (possibly multiple sequence numbers) so that the secondary can consume them * as-is. This is done only in the case when the primary is running on a GT.M version less than the GT.M version on secondary * side. Otherwise, the secondary takes the responsibility of doing the endian conversion. Note that the endian conversion happens * in-place. The below function is based on gtmrecv_process.c/repl_tr_endian_convert() */ static void repl_tr_endian_convert(repl_msg_ptr_t send_msgp, int send_tr_len, seq_num pre_read_seqno) { uchar_ptr_t buffp, jb; DEBUG_ONLY(uchar_ptr_t jstart;) int buflen, remaining_len, jlen, reclen, status, nodeflags_keylen, temp_val, keylen; jnl_record *rec; enum jnl_record_type rectype; jrec_suffix *suffixp; jnl_string *keystr; mstr_len_t *vallen_ptr; /* seq_num good_seqno; */ buffp = send_msgp->msg; buflen = send_msgp->len - REPL_MSG_HDRLEN; remaining_len = send_tr_len; /* QWASSIGN(good_seqno, seq_num_zero); */ while (0 < remaining_len) { jlen = buflen; jb = buffp; while (JREC_PREFIX_SIZE <= jlen) { DEBUG_ONLY(jstart = jb); rec = (jnl_record *)(jb); /* endian convert the prefix fields. Not all of the prefix fields are used by the secondary. Only rectype * and forwptr are needed. */ rectype = (enum jnl_record_type)rec->prefix.jrec_type; reclen = rec->prefix.forwptr; rec->prefix.forwptr = GTM_BYTESWAP_24(reclen); if (!IS_REPLICATED(rectype) || (0 == reclen) || (reclen > jlen) || (reclen > MAX_LOGI_JNL_REC_SIZE)) { assert(FALSE); status = -1; break; } assert(!IS_ZTP(rectype)); assert(IS_SET_KILL_ZKILL_ZTRIG_ZTWORM(rectype) || (JRT_TCOM == rectype) || (JRT_NULL == rectype)); /* endian convert the suffix fields. Only backptr needs endian conversion as the other field - suffix_code * is 8 bit. */ suffixp = ((jrec_suffix *)((unsigned char *)rec + reclen - JREC_SUFFIX_SIZE)); suffixp->backptr = GTM_BYTESWAP_24(suffixp->backptr); /* QWASSIGN(good_seqno, rec->jrec_null.jnl_seqno); */ /* update good_seqno */ rec->jrec_null.jnl_seqno = GTM_BYTESWAP_64(rec->jrec_null.jnl_seqno); /* At this point, we could have a TCOM or NULL or SET/KILL/ZKILL/ZTRIG type of record. * Assert that all of them have "strm_seqno" at the exact same offset so we can avoid * an if/then/else check on the record types in order to endian convert "strm_seqno". */ assert(&rec->jrec_null.strm_seqno == &rec->jrec_set_kill.strm_seqno); assert(&rec->jrec_null.strm_seqno == &rec->jrec_tcom.strm_seqno); rec->jrec_null.strm_seqno = GTM_BYTESWAP_64(rec->jrec_null.strm_seqno); if (IS_SET_KILL_ZKILL_ZTRIG_ZTWORM(rectype)) { keystr = (jnl_string *)&rec->jrec_set_kill.mumps_node; assert(keystr == (jnl_string *)&rec->jrec_ztworm.ztworm_str); assert(&rec->jrec_set_kill.update_num == &rec->jrec_ztworm.update_num); rec->jrec_set_kill.update_num = GTM_BYTESWAP_32(rec->jrec_set_kill.update_num); /* From V19 onwards, the 'length' field is divided into 8 bit 'nodeflags' and 24 bit 'length' * fields. */ keylen = keystr->length; nodeflags_keylen = *(jnl_str_len_t *)keystr; *(jnl_str_len_t *)keystr = GTM_BYTESWAP_32(nodeflags_keylen); if (IS_SET(rectype) || IS_ZTWORM(rectype)) { /* SET and ZTWORM records have a 'value' part which needs to be endian converted */ vallen_ptr = (mstr_len_t *)&keystr->text[keylen]; GET_MSTR_LEN(temp_val, vallen_ptr); temp_val = GTM_BYTESWAP_32(temp_val); PUT_MSTR_LEN(vallen_ptr, temp_val); /* The actual 'value' itself is a character array and hence needs no endian conversion */ } } else if (JRT_TCOM == rectype) { assert((unsigned char *)&rec->jrec_tcom.token_seq + SIZEOF(token_seq_t) == (unsigned char *)&rec->jrec_tcom.filler_short); /* endian convert num_participants */ rec->jrec_tcom.num_participants = GTM_BYTESWAP_16(rec->jrec_tcom.num_participants); } /* else records can only be JRT_NULL. The only relevant field in JRT_NULL is the sequence number which is * already endian converted. */ assert(jstart == jb); /* endian conversion should always happen in-place. */ jlen -= reclen; jb += reclen; } if ((-1 == status) || (0 != jlen)) { assert(FALSE); rts_error(VARLSTCNT(5) ERR_REPLXENDIANFAIL, 3, LEN_AND_LIT("Originating"), &pre_read_seqno); } /* move on to the next transaction */ remaining_len -= (buflen + REPL_MSG_HDRLEN); buffp += buflen; assert((REPL_TR_JNL_RECS == ((repl_msg_ptr_t)(buffp))->type) || (0 == remaining_len)); buflen = ((repl_msg_ptr_t)(buffp))->len - REPL_MSG_HDRLEN; buffp += REPL_MSG_HDRLEN; } if (0 != remaining_len) { rts_error(VARLSTCNT(5) ERR_REPLXENDIANFAIL, 3, LEN_AND_LIT("Originating"), &pre_read_seqno); assert(FALSE); } } /* The work-horse of the Source Server */ int gtmsource_process(void) { gtmsource_local_ptr_t gtmsource_local; jnlpool_ctl_ptr_t jctl; seq_num recvd_seqno, sav_read_jnl_seqno; struct sockaddr_in secondary_addr; seq_num recvd_jnl_seqno, tmp_read_jnl_seqno; int data_len, srch_status; unsigned char *msg_ptr; /* needed for REPL_{SEND,RECV}_LOOP */ int tosend_len, sent_len, sent_this_iter; /* needed for REPL_SEND_LOOP */ int torecv_len, recvd_len, recvd_this_iter; /* needed for REPL_RECV_LOOP */ int status; /* needed for REPL_{SEND,RECV}_LOOP */ int tot_tr_len, send_tr_len, remaining_len, pre_cmpmsglen; struct timeval poll_time; int recvd_msg_type, recvd_start_flags; uchar_ptr_t in_buff, out_buff, out_buffmsg; uint4 in_buflen, out_buflen, out_bufsiz; seq_num log_seqno, diff_seqno, pre_read_seqno, post_read_seqno, jnl_seqno; char err_string[1024]; boolean_t xon_wait_logged, prev_catchup, catchup, force_recv_check, already_communicated; double time_elapsed; seq_num resync_seqno, zqgblmod_seqno, filter_seqno; gd_region *reg, *region_top; sgmnt_addrs *csa; qw_num backlog_bytes, backlog_count, delta_sent_cnt, delta_data_sent, delta_msg_sent; long prev_msg_sent = 0; time_t prev_now = 0, save_now; int index; struct timeval poll_wait, poll_immediate; uint4 temp_ulong; unix_db_info *udi; repl_histinfo remote_histinfo, local_histinfo; int4 num_histinfo, max_epoch_interval; seq_num local_jnl_seqno, tmp_seqno; repl_msg_t xoff_ack, instnohist_msg, losttncomplete_msg; repl_msg_ptr_t send_msgp; repl_cmpmsg_ptr_t send_cmpmsgp; repl_start_reply_msg_ptr_t reply_msgp; boolean_t rollback_first, secondary_ahead, secondary_was_rootprimary; boolean_t intfilter_error, skip_last_histinfo_check, msg_is_cross_endian, retval; int semval, cmpret; uLongf cmpbuflen; int4 msghdrlen; Bytef *cmpbufptr; char histdetail[256]; gtm_time4_t tmp_time4; repl_heartbeat_msg_ptr_t heartbeat_msg; DEBUG_ONLY(uchar_ptr_t save_inbuff;) DEBUG_ONLY(uchar_ptr_t save_outbuff;) DCL_THREADGBL_ACCESS; SETUP_THREADGBL_ACCESS; assert((NULL != jnlpool.jnlpool_dummy_reg) && jnlpool.jnlpool_dummy_reg->open); DEBUG_ONLY( csa = &FILE_INFO(jnlpool.jnlpool_dummy_reg)->s_addrs; assert(!csa->hold_onto_crit); /* so we can do unconditional grab_lock/rel_lock */ ASSERT_VALID_JNLPOOL(csa); ) assert(REPL_MSG_HDRLEN == SIZEOF(jnldata_hdr_struct)); /* necessary for reading multiple transactions from jnlpool in * a single attempt */ jctl = jnlpool.jnlpool_ctl; gtmsource_local = jnlpool.gtmsource_local; gtmsource_msgp = NULL; gtmsource_msgbufsiz = MAX_REPL_MSGLEN; if (ZLIB_CMPLVL_NONE != gtm_zlib_cmp_level) gtmsource_cmpmsgp = NULL; assert(GTMSOURCE_POLL_WAIT < MAX_GTMSOURCE_POLL_WAIT); gtmsource_poll_wait.tv_sec = 0; gtmsource_poll_wait.tv_usec = GTMSOURCE_POLL_WAIT; poll_wait = gtmsource_poll_wait; gtmsource_poll_immediate.tv_sec = 0; gtmsource_poll_immediate.tv_usec = 0; poll_immediate = gtmsource_poll_immediate; gtmsource_init_sec_addr(&secondary_addr); gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION; /* Below is a simplistic representation of the state diagram of a source server. * * ------------------------------ * GTMSOURCE_START * ------------------------------ * | * | (startup state) * v * ------------------------------ * GTMSOURCE_WAITING_FOR_CONNECTION * ------------------------------ * | * | (gtmsource_est_conn) * v * ------------------------------ * GTMSOURCE_WAITING_FOR_RESTART * ------------------------------ * | * | (gtmsource_recv_restart) * v * ------------------------------ * GTMSOURCE_SEARCHING_FOR_RESTART * ------------------------------ * | * | (gtmsource_srch_restart) * v * ------------------------------ * GTMSOURCE_SENDING_JNLRECS <---------\ * ------------------------------ | * | | * | (receive REPL_XOFF) | * | (receive REPL_XOFF_ACK_ME) | * v | * ------------------------------ ^ * GTMSOURCE_WAITING_FOR_XON | * ------------------------------ | * | | * v (receive REPL_XON) | * | | * \--------------------->---------/ */ udi = FILE_INFO(jnlpool.jnlpool_dummy_reg); /* Before entering the loop find the max EPOCH interval (for use in lock waits) */ max_epoch_interval = 0; for (reg = gd_header->regions, region_top = gd_header->regions + gd_header->n_regions; reg < region_top; reg++) { csa = &FILE_INFO(reg)->s_addrs; if (max_epoch_interval < csa->hdr->epoch_interval) max_epoch_interval = csa->hdr->epoch_interval; } while (TRUE) { assert(!udi->grabbed_ftok_sem); gtmsource_stop_heartbeat(); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) { /* Ensure we don't hold any locks at this moment */ assert(process_id != gtmsource_local->gtmsource_srv_latch.u.parts.latch_pid); assert(!have_crit(CRIT_HAVE_ANY_REG)); /* checks both journal pool lock and database crit lock */ assert(FD_INVALID != gtmsource_sock_fd); if (FD_INVALID != gtmsource_sock_fd) { repl_log(gtmsource_log_fp, TRUE, TRUE, "Closing connection due to ONLINE ROLLBACK\n"); repl_close(>msource_sock_fd); SHORT_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_CLOSE_CONN); } jnl_seqno = jnlpool.jnlpool_ctl->jnl_seqno; repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL INFO - Current Jnlpool Seqno : %llu\n", jnl_seqno); repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL INFO - Last Seqno sent : %llu\n", gtmsource_local->read_jnl_seqno - 1); /* gtmsource_save_read_jnl_seqno is kept uptodate with gtmsource_local->read_addr and gtmsource_local->read * fields in gtmsource_onln_rlbk_clnup. But, gtmsource_local->read_jnl_seqno is still pointing to the last * sequence number that we sent to the receiver (which could have been rolled back now). We don't want to * continue with a stale value of read_jnl_seqno. So, set it to gtmsource_save_read_jnl_seqno which itself * is taken from the jnlpool_ctl->jnl_seqno right when we detected the online rollback. We could have set * this right when we set gtmsource_save_read_jnl_seqno but we don't do that because we want to print the * old value in the log file but we can't use repl_log/gtmsource_log_fp in gtmsource_onln_rlbk_clnup() as * it is bundled up as part of libgtmshr.so whereas repl_log is bundled in libmupip.a. */ gtmsource_local->read_jnl_seqno = gtmsource_save_read_jnl_seqno; repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL INFO - Source Server Read Seqno is now set to : %llu\n", gtmsource_local->read_jnl_seqno); gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION; assert(READ_FILE == gtmsource_local->read_state); gtmsource_ctl_close(); /* can't rely on the journal files anymore since rollback could have touched them */ } if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) { gtmsource_start_jnl_release_timer(); gtmsource_est_conn(&secondary_addr); gtmsource_stop_jnl_release_timer(); if (GTMSOURCE_CHANGING_MODE == gtmsource_state) return (SS_NORMAL); repl_source_data_sent = repl_source_msg_sent = repl_source_cmp_sent = 0; repl_source_lastlog_data_sent = 0; repl_source_lastlog_msg_sent = 0; gtmsource_alloc_msgbuff(MAX_REPL_MSGLEN); gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_RESTART; recvd_start_flags = START_FLAG_NONE; repl_source_prev_log_time = time(NULL); } if (GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state && SS_NORMAL != (status = gtmsource_recv_restart(&recvd_seqno, &recvd_msg_type, &recvd_start_flags))) { if (EREPL_RECV == repl_errno) { if (REPL_CONN_RESET(status)) { /* Connection reset */ repl_log(gtmsource_log_fp, TRUE, TRUE, "Connection reset while receiving restart SEQNO. Status = %d ; %s\n", status, STRERROR(status)); repl_close(>msource_sock_fd); SHORT_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_CLOSE_CONN); gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION; continue; } else { SNPRINTF(err_string, SIZEOF(err_string), "Error receiving RESTART SEQNO. Error in recv : %s", STRERROR(status)); rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_STRING(err_string)); } } else if (EREPL_SEND == repl_errno) { if (REPL_CONN_RESET(status)) { repl_log(gtmsource_log_fp, TRUE, TRUE, "Connection reset while sending XOFF_ACK due to possible update process shutdown. " "Status = %d ; %s\n", status, STRERROR(status)); repl_close(>msource_sock_fd); SHORT_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_CLOSE_CONN); gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION; continue; } SNPRINTF(err_string, SIZEOF(err_string), "Error sending XOFF_ACK_ME message. Error in send : %s", STRERROR(status)); rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_STRING(err_string)); } else if (EREPL_SELECT == repl_errno) { SNPRINTF(err_string, SIZEOF(err_string), "Error receiving RESTART SEQNO/sending XOFF_ACK_ME. " "Error in select : %s", STRERROR(status)); rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_STRING(err_string)); } } if (GTMSOURCE_CHANGING_MODE == gtmsource_state) return (SS_NORMAL); /* Connection might have been closed if "gtmsource_recv_restart" got an unexpected message. In that case * re-establish the same by continuing to the beginning of this loop. */ if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) continue; assert(REPL_PROTO_VER_MULTISITE <= remote_side->proto_ver); assert((GTMSOURCE_SEARCHING_FOR_RESTART == gtmsource_state) || (GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state)); /* Receiver runs on a version of GT.M that supports multi-site capability */ /* If gtmsource_state == GTMSOURCE_SEARCHING_FOR_RESTART, we have already communicated with the * receiver and hence checked the instance info so no need to do it again. */ if (GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state) { /* Get replication instance info */ DEBUG_ONLY(secondary_was_rootprimary = -1;) /* Note: As part of the REPL_INSTINFO message, the receiver could be sending a non-zero "strm_jnl_seqno" * in some cases. If so, it will override "recvd_seqno" we saw before in the REPL_START_JNL_SEQNO message. */ if (!gtmsource_get_instance_info(&secondary_was_rootprimary, &recvd_seqno)) { if (GTMSOURCE_CHANGING_MODE == gtmsource_state) return (SS_NORMAL); else if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) continue; else { /* Got a REPL_XOFF_ACK_ME from the receiver. Restart the initial handshake */ assert(GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state); continue; } } assert((FALSE == secondary_was_rootprimary) || (TRUE == secondary_was_rootprimary)); } rollback_first = FALSE; secondary_ahead = FALSE; GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) continue; local_jnl_seqno = jctl->jnl_seqno; rel_lock(jnlpool.jnlpool_dummy_reg); /* Take care to set the flush parameter in repl_log calls below to FALSE until at least the first message * gets sent back. This is so the fetchresync rollback on the other side does not timeout before receiving * a response. */ assert(0 == GET_STRM_INDEX(recvd_seqno)); assert(0 == GET_STRM_INDEX(local_jnl_seqno)); repl_log(gtmsource_log_fp, TRUE, FALSE, "Current Journal Seqno of the instance is %llu [0x%llx]\n", local_jnl_seqno, local_jnl_seqno); if (recvd_seqno > local_jnl_seqno) { /* Secondary journal seqno is greater than that of the Primary. We know it is ahead of the primary. */ secondary_ahead = TRUE; repl_log(gtmsource_log_fp, TRUE, FALSE, "Secondary instance journal seqno %llu [0x%llx] is greater than Primary " "instance journal seqno %llu [0x%llx]\n", recvd_seqno, recvd_seqno, local_jnl_seqno, local_jnl_seqno); /* Since the secondary is at least multi-site, the determination of the rollback seqno involves comparing * the histinfo records between the primary and secondary starting down from "local_jnl_seqno-1" * (done below). In either case, the secondary has to roll back to at most "local_jnl_seqno". * Reset "recvd_seqno" to this number given that we have already recorded that the secondary is * ahead of the primary. */ recvd_seqno = local_jnl_seqno; } /* Before setting "next_histinfo_seqno", check if we have at least one histinfo record in the replication instance * file. The only case when there can be no histinfo records is if this instance is a propagating primary. Assert * that. In this case, wait for this instance's primary to send the first histinfo record before setting the * next_histinfo_seqno. Note that we are fetching the value of "num_histinfo" without holding a lock on the instance * file but that is ok since all we care about is if it is 0 or not. We do not rely on the actual value. */ num_histinfo = jnlpool.repl_inst_filehdr->num_histinfo; assert(0 <= num_histinfo); assert(num_histinfo || jctl->upd_disabled); gtmsource_local->next_histinfo_num = -1;/* Initial value. Reset by the call to "gtmsource_set_next_histinfo_seqno" * invoked in turn by "gtmsource_send_new_histrec" down below */ if (jctl->upd_disabled && !num_histinfo) { /* Wait for corresponding primary to send a new histinfo record and the receiver server on this instance * to write that to the replication instance file. */ assert(-1 == gtmsource_local->next_histinfo_num); repl_log(gtmsource_log_fp, TRUE, TRUE, "Source server waiting for first history record to be written by " "update process\n"); do { SHORT_SLEEP(GTMSOURCE_WAIT_FOR_FIRSTHISTINFO); gtmsource_poll_actions(FALSE); if (GTMSOURCE_CHANGING_MODE == gtmsource_state) return (SS_NORMAL); num_histinfo = jnlpool.repl_inst_filehdr->num_histinfo; if (num_histinfo) /* Number of histinfos is non-zero */ break; } while (TRUE); repl_log(gtmsource_log_fp, TRUE, TRUE, "First history record written by update process. Source server proceeding.\n"); } /* Now get the latest histinfo record from the secondary. There are a few exceptions though. * 1) If we came here because of a BAD_TRANS or CMP2UNCMP message from the receiver server. * In this case, we have already been communicating with the receiver so no need to * compare the histinfo record between primary and secondary. * 2) If receiver server was started with -UPDATERESYNC and receiver is running pre-V55000. * In this case there is no history record on the receiver side to compare against. * In case the receiver is post-V55000, the -UPDATERESYNC would have required an instance * file name as the value which would be used towards history record verification. * 3) If receiver server was started with -UPDATERESYNC and receiver is >= V55000 and at a seqno * which is EQUAL to the earliest seqno for which we have a history record on the primary. * We have no way of verifying histories since we definitely dont have the history record * for the receiver side seqno. Since -updateresync was used, assume they are in sync and * start replicating from the earliest seqno for which we have a history record on the primary. * 4) If recvd_seqno is 1. In this case, the receiver instance has been created afresh so its instance * file is empty and we are guaranteed there is nothing to compare. So no point requesting it. * Besides, this is a very common situation in practice that requiring -updateresync in this * case seems user-unfriendly so we will let this one go by without a -updateresync particularly * because there is no harm that can happen by allowing two such instances to connect/replicate. */ assert(0 != recvd_seqno); if (1 == recvd_seqno) skip_last_histinfo_check = TRUE; else { if ((GTMSOURCE_WAITING_FOR_RESTART != gtmsource_state) && already_communicated) skip_last_histinfo_check = TRUE; else if (START_FLAG_UPDATERESYNC & recvd_start_flags) { repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL_START_JNL_SEQNO message has " "START_FLAG_UPDATERESYNC bit set\n"); if (REPL_PROTO_VER_SUPPLEMENTARY > remote_side->proto_ver) skip_last_histinfo_check = TRUE; else { assert(jnlpool.repl_inst_filehdr->num_histinfo); /* should be at least 1 history record */ /* If -updateresync is specified and receiver instance seqno is exactly equal to the * start_seqno of the earliest history record in the instance file, then skip last * histinfo check. Note that in case both source and receiver instances are supplementary, * we should be looking at the 0th stream only. Even in that case, we are guaranteed that * the 0th history record in the instance file corresponds to the 0th stream. So it is * safe to look at the start_seqno of just the 0th history record in all cases. */ GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) continue; status = repl_inst_histinfo_get(0, &local_histinfo); assert(0 == status); /* Since we pass histinfo_num of 0 which is >=0 and < num_histinfo */ rel_lock(jnlpool.jnlpool_dummy_reg); if (local_histinfo.start_seqno == recvd_seqno) skip_last_histinfo_check = TRUE; else skip_last_histinfo_check = FALSE; } } else skip_last_histinfo_check = FALSE; } if (!skip_last_histinfo_check) { /* Find histinfo record in the local instance file corresponding to seqno "recvd_seqno-1" */ GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) continue; assert(recvd_seqno <= local_jnl_seqno); assert(recvd_seqno <= jctl->jnl_seqno); assert((INVALID_SUPPL_STRM == strm_index) || (0 == strm_index)); status = repl_inst_histinfo_find_seqno(recvd_seqno, strm_index, &local_histinfo); rel_lock(jnlpool.jnlpool_dummy_reg); assert((0 != status) || (local_histinfo.start_seqno < recvd_seqno)); if (0 != status) { /* If recvd_seqno is the earliest history record's start_seqno and -udpateresync was * specified, assume the two instances are in sync. Otherwise issue error and close connection. * Send this error status to the receiver server before closing the connection. * This way the receiver will know to shut down rather than loop back trying to * reconnect. This avoids an infinite loop of connection open and closes * between the source server and receiver server. */ assert(ERR_REPLINSTNOHIST == status); /* only error returned by "repl_inst_histinfo_find_seqno" */ assert((INVALID_HISTINFO_NUM == local_histinfo.histinfo_num) || (local_histinfo.start_seqno >= recvd_seqno)); if (!(START_FLAG_UPDATERESYNC & recvd_start_flags) || (INVALID_HISTINFO_NUM == local_histinfo.histinfo_num) || (local_histinfo.start_seqno > recvd_seqno)) { /* recvd_seqno is PRIOR to the starting seqno of the instance file. * In that case, issue error and close the connection. */ SPRINTF(histdetail, "seqno "INT8_FMT" "INT8_FMTX, recvd_seqno - 1, recvd_seqno - 1); gtm_putmsg(VARLSTCNT(6) ERR_REPLINSTNOHIST, 4, LEN_AND_STR(histdetail), LEN_AND_STR(udi->fn)); instnohist_msg.type = REPL_INST_NOHIST; instnohist_msg.len = MIN_REPL_MSGLEN; memset(&instnohist_msg.msg[0], 0, SIZEOF(instnohist_msg.msg)); gtmsource_repl_send((repl_msg_ptr_t)&instnohist_msg, "REPL_INST_NOHIST", MAX_SEQNO, INVALID_SUPPL_STRM); repl_log(gtmsource_log_fp, TRUE, TRUE, "Connection reset due to above REPLINSTNOHIST error\n"); repl_close(>msource_sock_fd); SHORT_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_CLOSE_CONN); gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION; continue; } assert((0 == local_histinfo.histinfo_num) && (local_histinfo.start_seqno == recvd_seqno)); } if (local_histinfo.start_seqno < recvd_seqno) { /* Find histinfo record in the remote instance file corresponding to seqno "recvd_seqno-1" */ retval = gtmsource_get_remote_histinfo(recvd_seqno, &remote_histinfo); if (retval) { assert(remote_histinfo.start_seqno < recvd_seqno); /* Check if primary and secondary have same histinfo for "recvd_seqno-1" */ rollback_first = !gtmsource_is_histinfo_identical(&remote_histinfo, &local_histinfo, recvd_seqno, OK_TO_LOG_TRUE); /* If local and remote sides are supplementary (i.e. P->Q replication), verify each * stream level history as well. Do this only if the remote side is a receiver server * (i.e. not rollback) and if we still intend on sending a REPL_WILL_RESTART_WITH_INFO * message. */ assert(this_side->is_supplementary == jnlpool.repl_inst_filehdr->is_supplementary); if (this_side->is_supplementary && remote_side->is_supplementary && (REPL_START_JNL_SEQNO == recvd_msg_type) && !rollback_first && !secondary_ahead) retval = gtmsource_check_remote_strm_histinfo(recvd_seqno, &rollback_first); } if (!retval) { if (GTMSOURCE_CHANGING_MODE == gtmsource_state) return (SS_NORMAL); else if ((GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) || (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)) continue; else { /* Got a REPL_XOFF_ACK_ME from receiver. Restart the initial handshake */ assert(GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state); continue; } } } } QWASSIGN(sav_read_jnl_seqno, gtmsource_local->read_jnl_seqno); reply_msgp = (repl_start_reply_msg_ptr_t)gtmsource_msgp; memset(reply_msgp, 0, SIZEOF(*reply_msgp)); /* to identify older releases in the future */ reply_msgp->len = MIN_REPL_MSGLEN; reply_msgp->proto_ver = REPL_PROTO_VER_THIS; reply_msgp->node_endianness = NODE_ENDIANNESS; reply_msgp->is_supplementary = jnlpool.repl_inst_filehdr->is_supplementary; assert((1 != recvd_seqno) || !rollback_first); if ((GTMSOURCE_SEARCHING_FOR_RESTART == gtmsource_state) || (REPL_START_JNL_SEQNO == recvd_msg_type)) { gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_SEARCHING_FOR_RESTART; /* If the last histinfo record in both instances are NOT the same ("rollback_first" is TRUE) * (possible only if the secondary is multi-site), or if secondary is ahead of the primary * ("secondary_ahead" is TRUE) we do want the secondary to rollback first. Issue message to * do rollback fetchresync. There is one exception though. And that is if -NORESYNC was * specified on the receiver side. In this case, determine the resync/common point by comparing * local and remote histinfo records from the tail of the instance file until we reach * one seqno whose histinfo information is identical in both. * Use this as the common point to send a REPL_WILL_RESTART_WITH_INFO message. */ if (!rollback_first && !secondary_ahead) resync_seqno = recvd_seqno; else if (START_FLAG_NORESYNC & recvd_start_flags) { repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL_START_JNL_SEQNO message has " "START_FLAG_NORESYNC bit set\n"); assert(!skip_last_histinfo_check); assert(1 != recvd_seqno); if (!rollback_first) { assert(secondary_ahead); assert(recvd_seqno == local_jnl_seqno); /* The primary and secondary are in sync as of "recvd_seqno" the jnl seqno of the * primary. So that is the common point. Send it across. */ resync_seqno = recvd_seqno; } else { resync_seqno = gtmsource_find_resync_seqno(&local_histinfo, &remote_histinfo); assert((MAX_SEQNO != resync_seqno) || (GTMSOURCE_CHANGING_MODE == gtmsource_state) || (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) || (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)); rollback_first = FALSE; } } else { /* Ask secondary to issue a fetchresync rollback */ repl_log(gtmsource_log_fp, TRUE, FALSE, "Secondary instance needs to first do MUPIP JOURNAL ROLLBACK FETCHRESYNC\n"); resync_seqno = local_jnl_seqno; rollback_first = TRUE; } if (MAX_SEQNO != resync_seqno) { QWASSIGN(*(seq_num *)&reply_msgp->start_seqno[0], resync_seqno); if (!rollback_first) { GRAB_GTMSOURCE_SRV_LATCH(gtmsource_local, max_epoch_interval); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) continue; srch_status = gtmsource_srch_restart(resync_seqno, recvd_start_flags); rel_gtmsource_srv_latch(>msource_local->gtmsource_srv_latch); assert(resync_seqno == gtmsource_local->read_jnl_seqno); assert(SS_NORMAL == srch_status); reply_msgp->type = REPL_WILL_RESTART_WITH_INFO; reply_msgp->jnl_ver = this_side->jnl_ver; temp_ulong = (0 == this_side->is_std_null_coll) ? START_FLAG_NONE : START_FLAG_COLL_M; GTMTRIG_ONLY( assert(this_side->trigger_supported); temp_ulong |= START_FLAG_TRIGGER_SUPPORT; ) PUT_ULONG(reply_msgp->start_flags, temp_ulong); recvd_start_flags = START_FLAG_NONE; gtmsource_repl_send((repl_msg_ptr_t)reply_msgp, "REPL_WILL_RESTART_WITH_INFO", resync_seqno, INVALID_SUPPL_STRM); } else { /* Secondary needs to first do FETCHRESYNC rollback to synchronize with primary */ reply_msgp->type = REPL_ROLLBACK_FIRST; gtmsource_repl_send((repl_msg_ptr_t)reply_msgp, "REPL_ROLLBACK_FIRST", resync_seqno, INVALID_SUPPL_STRM); } } } else { /* REPL_FETCH_RESYNC received and state is WAITING_FOR_RESTART */ if (rollback_first || secondary_ahead) { /* Primary and Secondary are currently not in sync */ if (!rollback_first) { /* We know the secondary is ahead of the primary in terms of journal seqno but the last * histinfo records are identical. This means that the secondary is in sync with the * primary until the primary's journal seqno ("local_jnl_seqno") which should be the new * resync seqno. */ resync_seqno = local_jnl_seqno; } else { /* Determine the resync seqno between this primary and secondary by comparing * local and remote histinfo records from the tail of the instance file until we reach * one seqno whose histinfo information is identical in both. */ assert(1 != recvd_seqno); resync_seqno = gtmsource_find_resync_seqno(&local_histinfo, &remote_histinfo); assert((MAX_SEQNO != resync_seqno) || (GTMSOURCE_CHANGING_MODE == gtmsource_state) || (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) || (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)); } } else { /* Primary and Secondary are in sync upto "recvd_seqno". Send it back as the new resync seqno. */ resync_seqno = recvd_seqno; } if (MAX_SEQNO != resync_seqno) { assert(GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state && REPL_FETCH_RESYNC == recvd_msg_type); reply_msgp->type = REPL_RESYNC_SEQNO; QWASSIGN(*(seq_num *)&reply_msgp->start_seqno[0], resync_seqno); gtmsource_repl_send((repl_msg_ptr_t)reply_msgp, "REPL_RESYNC_SEQNO", resync_seqno, INVALID_SUPPL_STRM); } } if (GTMSOURCE_CHANGING_MODE == gtmsource_state) return (SS_NORMAL); /* "gtmsource_repl_send" or "gtmsource_find_resync_seqno" did not complete */ if ((GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) || (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)) continue; /* "gtmsource_repl_send" or "gtmsource_find_resync_seqno" did not complete */ assert(MAX_SEQNO != resync_seqno); /* After having established connection, initialize a few fields in the gtmsource_local * structure and flush those changes to the instance file on disk. */ GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) continue; gtmsource_local->connect_jnl_seqno = jctl->jnl_seqno; gtmsource_local->send_losttn_complete = jctl->send_losttn_complete; /* Now that "connect_jnl_seqno" has been updated, flush it to corresponding gtmsrc_lcl on disk */ repl_inst_flush_gtmsrc_lcl(); /* this requires the jnlpool lock to be held */ rel_lock(jnlpool.jnlpool_dummy_reg); if (REPL_WILL_RESTART_WITH_INFO != reply_msgp->type) { assert(reply_msgp->type == REPL_RESYNC_SEQNO || reply_msgp->type == REPL_ROLLBACK_FIRST); if ((REPL_RESYNC_SEQNO == reply_msgp->type) && secondary_was_rootprimary) { repl_log(gtmsource_log_fp, TRUE, TRUE, "Sent REPL_RESYNC_SEQNO message with SEQNO %llu [0x%llx]\n", (*(seq_num *)&reply_msgp->start_seqno[0]), (*(seq_num *)&reply_msgp->start_seqno[0])); region_top = gd_header->regions + gd_header->n_regions; assert(NULL != jnlpool.jnlpool_dummy_reg); GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) continue; zqgblmod_seqno = jctl->max_zqgblmod_seqno; if (0 == zqgblmod_seqno) { /* If zqgblmod_seqno in all file headers is 0, it implies that this is the first * FETCHRESYNC rollback after the most recent MUPIP REPLIC -LOSTTNCOMPLETE command. * Therefore reset zqgblmod_seqno to the rollback seqno. If not the first rollback, * then zqgblmod_seqno will be reset only if the new rollback seqno is lesser * than the current value. */ zqgblmod_seqno = MAX_SEQNO; /* actually 0xFFFFFFFFFFFFFFFF (max possible seqno) */ /* Reset any pending MUPIP REPLIC -SOURCE -LOSTTNCOMPLETE */ jctl->send_losttn_complete = FALSE; gtmsource_local->send_losttn_complete = jctl->send_losttn_complete; } rel_lock(jnlpool.jnlpool_dummy_reg); REPL_DPRINT2("BEFORE FINDING RESYNC - zqgblmod_seqno is %llu", zqgblmod_seqno); REPL_DPRINT2(", curr_seqno is %llu\n", jctl->jnl_seqno); if (zqgblmod_seqno > resync_seqno) { /* reset "zqgblmod_seqno" and "zqgblmod_tn" in all fileheaders to "resync_seqno" */ if (SS_NORMAL != gtmsource_update_zqgblmod_seqno_and_tn(resync_seqno)) { assert(GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) continue; } } } /* Could send a REPL_CLOSE_CONN message here */ /* It is expected that on receiving this msg, the Receiver Server will break the connection and exit. */ repl_close(>msource_sock_fd); LONG_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_TO_QUIT); /* may not be needed after REPL_CLOSE_CONN is sent */ gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION; continue; } /* Now that REPL_WILL_RESTART_WITH_INFO message has been sent, if compression of the replication stream is * requested, check if the receiver server supports ability to decompress. Dont do this if this receiver has * previously sent a REPL_CMP2UNCMP message. */ gtmsource_local->repl_zlib_cmp_level = repl_zlib_cmp_level = ZLIB_CMPLVL_NONE; /* no compression by default */ if (!gtmsource_received_cmp2uncmp_msg && (ZLIB_CMPLVL_NONE != gtm_zlib_cmp_level)) { if (REPL_PROTO_VER_MULTISITE_CMP <= remote_side->proto_ver) { /* Receiver server is running a version of GT.M that supports compression of replication stream. * Send test message with compressed data to check if it is able to decompress properly. If so, * enable compression on the replication pipe. Compression level set in repl_zlib_cmp_level. */ if (!gtmsource_get_cmp_info(&repl_zlib_cmp_level)) { if (GTMSOURCE_CHANGING_MODE == gtmsource_state) return (SS_NORMAL); else if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) continue; else { /* Got a REPL_XOFF_ACK_ME from receiver. Restart the initial handshake */ assert(GTMSOURCE_WAITING_FOR_RESTART == gtmsource_state); continue; } } /* Note down replication cmp_level in this source-server specific structure in journal pool */ gtmsource_local->repl_zlib_cmp_level = repl_zlib_cmp_level; } else { repl_log(gtmsource_log_fp, TRUE, FALSE, "Receiver server does not support compressed data on the replication pipe\n"); repl_log(gtmsource_log_fp, TRUE, FALSE, "Defaulting to NO compression\n"); } } if (QWLT(gtmsource_local->read_jnl_seqno, sav_read_jnl_seqno) && (NULL != repl_ctl_list)) { /* The journal files may have been positioned ahead of the read_jnl_seqno for the next read. * Indicate that they have to be repositioned into the past. */ assert(READ_FILE == gtmsource_local->read_state); gtmsource_set_lookback(); } poll_time = poll_immediate; gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_SENDING_JNLRECS; assert(1 <= gtmsource_local->read_jnl_seqno); /* Now that "gtmsource_local->read_jnl_seqno" is initialized, ensure the first send gets logged. */ gtmsource_reinit_logseqno(); gtmsource_init_heartbeat(); /* Internal filters are needed as long as the filter format of the originating side is greater or equal to the * filter format of the secondary side */ if ((this_side->jnl_ver >= remote_side->jnl_ver) && (IF_NONE != repl_filter_cur2old[remote_side->jnl_ver - JNL_VER_EARLIEST_REPL])) { assert(IF_INVALID != repl_filter_cur2old[remote_side->jnl_ver - JNL_VER_EARLIEST_REPL]); assert(IF_INVALID != repl_filter_old2cur[remote_side->jnl_ver - JNL_VER_EARLIEST_REPL]); /* reverse transformation should exist */ assert(IF_NONE != repl_filter_old2cur[remote_side->jnl_ver - JNL_VER_EARLIEST_REPL]); if (this_side->is_std_null_coll != remote_side->is_std_null_coll) remote_side->null_subs_xform = (this_side->is_std_null_coll ? STDNULL_TO_GTMNULL_COLL : GTMNULL_TO_STDNULL_COLL); else remote_side->null_subs_xform = FALSE; gtmsource_filter |= INTERNAL_FILTER; /* Any time the ^#t global format version is bumped, the below assert will trip. This way, anyone who bumps * the trigger label ensures that the internal filter routines in repl_filter.c are accordingly changed to * downgrade triggers before sending them across to a receiver that understands ONLY the prior ^#t format. */ assert(0 == MEMCMP_LIT(HASHT_GBL_CURLABEL, "2")); gtmsource_alloc_filter_buff(gtmsource_msgbufsiz); } else { gtmsource_filter &= ~INTERNAL_FILTER; if (NO_FILTER == gtmsource_filter) gtmsource_free_filter_buff(); } catchup = FALSE; force_recv_check = TRUE; xon_wait_logged = FALSE; /* Flush "gtmsource_local->read_jnl_seqno" to disk right now. * This will serve as a reference point for next timed flush to occur. */ gtmsource_flush_fh(gtmsource_local->read_jnl_seqno); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) continue; gtmsource_local->send_new_histrec = TRUE; /* Send new histinfo unconditionally at start of connection */ gtmsource_local->next_histinfo_seqno = MAX_SEQNO; /* Initial value. Reset by "gtmsource_send_new_histrec" below */ assert(-1 == gtmsource_local->next_histinfo_num); while (TRUE) { gtmsource_poll_actions(TRUE); if (GTMSOURCE_CHANGING_MODE == gtmsource_state) return (SS_NORMAL); if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) break; if (gtmsource_local->send_losttn_complete) { /* Send LOSTTNCOMPLETE across to the secondary and reset flag to FALSE */ losttncomplete_msg.type = REPL_LOSTTNCOMPLETE; losttncomplete_msg.len = MIN_REPL_MSGLEN; gtmsource_repl_send((repl_msg_ptr_t)&losttncomplete_msg, "REPL_LOSTTNCOMPLETE", MAX_SEQNO, INVALID_SUPPL_STRM); GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) break; /* the outerloop will continue */ gtmsource_local->send_losttn_complete = FALSE; rel_lock(jnlpool.jnlpool_dummy_reg); } if (gtmsource_local->send_new_histrec) { /* We are at the beginning of a new histinfo record boundary. Send a REPL_HISTREC message * before sending journal records for seqnos corresponding to this histinfo. */ assert(REPL_PROTO_VER_MULTISITE <= remote_side->proto_ver); /* Remote version supports multi-site functionality. Send REPL_HISTREC and friends */ gtmsource_send_new_histrec(); if (GTMSOURCE_CHANGING_MODE == gtmsource_state) return (SS_NORMAL); if ((GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) || (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state)) break; assert(FALSE == gtmsource_local->send_new_histrec); } /* If the backlog is high, we want to avoid communication overhead as much as possible. We switch * our communication mode to *catchup* mode, wherein we don't wait for the pipe to become ready to * send. Rather, we assume the pipe is ready for sending. The risk is that the send may block if * the pipe is not ready for sending. In the user's perspective, the risk is that the source server * may not respond to administrative actions such as "change log", "shutdown" (although mupip stop * would work). */ pre_read_seqno = gtmsource_local->read_jnl_seqno; prev_catchup = catchup; assert(jctl->write_addr >= gtmsource_local->read_addr); backlog_bytes = jctl->write_addr - gtmsource_local->read_addr; GRAB_LOCK(jnlpool.jnlpool_dummy_reg, HANDLE_CONCUR_ONLINE_ROLLBACK); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) break; /* the outerloop will continue */ jnl_seqno = jctl->jnl_seqno; rel_lock(jnlpool.jnlpool_dummy_reg); assert(jnl_seqno >= pre_read_seqno - 1); /* jnl_seqno >= pre_read_seqno is the most common case; * see gtmsource_readpool() for when the rare case can occur */ backlog_count = (jnl_seqno >= pre_read_seqno) ? (jnl_seqno - pre_read_seqno) : 0; catchup = (BACKLOG_BYTES_THRESHOLD <= backlog_bytes || BACKLOG_COUNT_THRESHOLD <= backlog_count); if (!prev_catchup && catchup) /* transition from non catchup to catchup */ { repl_log(gtmsource_log_fp, TRUE, TRUE, "Source server entering catchup mode at Seqno %llu " "[0x%llx] : Current backlog %llu [0x%llx] : Backlog size in journal pool %llu [0x%llx] " "bytes\n", pre_read_seqno, pre_read_seqno, backlog_count, backlog_count, backlog_bytes, backlog_bytes); prev_now = gtmsource_now; prev_msg_sent = repl_source_msg_sent; force_recv_check = TRUE; } else if (prev_catchup && !catchup) /* transition from catchup to non catchup */ { repl_log(gtmsource_log_fp, TRUE, TRUE, "Source server returning to regular mode from catchup mode " "at Seqno %llu [0x%llx] : Current backlog %llu [0x%llx] : Backlog size in journal pool " " %llu [0x%llx] bytes\n", pre_read_seqno, pre_read_seqno, backlog_count, backlog_count, backlog_bytes, backlog_bytes); if (gtmsource_msgbufsiz - MAX_REPL_MSGLEN > 2 * OS_PAGE_SIZE) {/* We have expanded the buffer by too much (could have been avoided had we sent one transaction * at a time while reading from journal files); let's revert back to our initial buffer size. * If we don't reduce our buffer, it is possible that the buffer keeps growing (while reading * from journal file) thus making the size of sends while reading from journal pool very * large (> 1 MB). Better to do some house keeping. We will force an expansion if the transaction * size dictates it. Ideally, this must be done while switching reading back from files to * pool, but we can't afford to free the buffer until we sent the transaction out. That apart, * let's wait for some breathing time to do house keeping. In catchup mode, we intend to keep * the send size large. */ gtmsource_free_filter_buff(); gtmsource_free_msgbuff(); gtmsource_alloc_msgbuff(MAX_REPL_MSGLEN); /* will also allocate filter buffer */ } } /* Check if receiver sent us any control message. * Typically, the traffic from receiver to source is very low compared to traffic in the other direction. * More often than not, there will be nothing on the pipe to receive. Ideally, we should let TCP * notify us when there is data on the pipe (async I/O on Unix and VMS). We are not there yet. Until then, * we use heuristics - attempt receive every GTMSOURCE_SENT_THRESHOLD_FOR_RECV bytes of sent data, and * every heartbeat period, OR whenever we want to force a check */ if ((GTMSOURCE_SENDING_JNLRECS != gtmsource_state) || !catchup || prev_now != (save_now = gtmsource_now) || (GTMSOURCE_SENT_THRESHOLD_FOR_RECV <= (repl_source_msg_sent - prev_msg_sent)) || force_recv_check) { REPL_EXTRA_DPRINT2("gtmsource_process: receiving because : %s\n", (GTMSOURCE_SENDING_JNLRECS != gtmsource_state) ? "state is not SENDING_JNLRECS" : !catchup ? "not in catchup mode" : (prev_now != save_now) ? "heartbeat interval passed" : (GTMSOURCE_SENT_THRESHOLD_FOR_RECV <= repl_source_msg_sent - prev_msg_sent) ? "sent bytes threshold for recv crossed" : "force recv check"); REPL_EXTRA_DPRINT6("gtmsource_state : %d prev_now : %ld gtmsource_now : %ld repl_source_msg_sent " ": %ld prev_msg_sent : %ld\n", gtmsource_state, prev_now, save_now, repl_source_msg_sent, prev_msg_sent); REPL_RECV_LOOP(gtmsource_sock_fd, gtmsource_msgp, MIN_REPL_MSGLEN, FALSE, &poll_time) { if (0 == recvd_len) /* nothing received in the first attempt, let's try again later */ break; gtmsource_poll_actions(TRUE); if (GTMSOURCE_CHANGING_MODE == gtmsource_state) return (SS_NORMAL); if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) break; } REPL_EXTRA_DPRINT3("gtmsource_process: %d received, type is %d\n", recvd_len, (0 != recvd_len) ? gtmsource_msgp->type : -1); if (GTMSOURCE_SENDING_JNLRECS == gtmsource_state && catchup) { if (prev_now != save_now) prev_now = save_now; else if (GTMSOURCE_SENT_THRESHOLD_FOR_RECV <= repl_source_msg_sent - prev_msg_sent) { /* do not set to repl_source_msg_sent; increment by GTMSOURCE_SENT_THRESHOLD_FOR_RECV * instead so that we force recv every GTMSOURCE_SENT_THRESHOLD_FOR_RECV bytes */ prev_msg_sent += GTMSOURCE_SENT_THRESHOLD_FOR_RECV; } } } else { /* behave as if there was nothing to be read */ status = SS_NORMAL; recvd_len = 0; } force_recv_check = FALSE; if (SS_NORMAL == status && 0 != recvd_len) { /* Process the received control message */ assert(MIN_REPL_MSGLEN == recvd_len); /* One is not always guaranteed the received message is in source native endian format. * See endianness related comments in gtmsource_recv_restart for why. So be safe and handle * it just like how gtmsource_recv_restart does. The below check works as all messages we * expect at this point have a fixed length of MIN_REPL_MSGLEN. */ msg_is_cross_endian = (((unsigned)MIN_REPL_MSGLEN < (unsigned)gtmsource_msgp->len) && ((unsigned)MIN_REPL_MSGLEN == GTM_BYTESWAP_32((unsigned)gtmsource_msgp->len))); if (msg_is_cross_endian) { gtmsource_msgp->type = GTM_BYTESWAP_32(gtmsource_msgp->type); gtmsource_msgp->len = GTM_BYTESWAP_32(gtmsource_msgp->len); } assert(MIN_REPL_MSGLEN == gtmsource_msgp->len); assert(remote_side->endianness_known); /* Even though we know the endianness of the remote side at this point, we use * msg_is_cross_endian as it should be a safer alternative in case of messages * like REPL_XOFF_ACK_ME where the receiver (particularly pre-V55000 versions) * is not careful to send in a consistent endian format across versions. So we * use the endianness of this particular message rather than the endianness of * the connection. */ switch(gtmsource_msgp->type) { case REPL_XOFF: case REPL_XOFF_ACK_ME: gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_XON; poll_time = poll_wait; repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL_XOFF/REPL_XOFF_ACK_ME received. Send stalled...\n"); xon_wait_logged = FALSE; if (REPL_XOFF_ACK_ME == gtmsource_msgp->type) { xoff_ack.type = REPL_XOFF_ACK; tmp_seqno = *(seq_num *)>msource_msgp->msg[0]; if (msg_is_cross_endian) tmp_seqno = GTM_BYTESWAP_64(tmp_seqno); *(seq_num *)&xoff_ack.msg[0] = tmp_seqno; xoff_ack.len = MIN_REPL_MSGLEN; gtmsource_repl_send((repl_msg_ptr_t)&xoff_ack, "REPL_XOFF_ACK", MAX_SEQNO, INVALID_SUPPL_STRM); if (GTMSOURCE_CHANGING_MODE == gtmsource_state) return (SS_NORMAL); /* "gtmsource_repl_send" did not complete */ if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) break; /* "gtmsource_repl_send" did not complete */ } break; case REPL_XON: gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_SENDING_JNLRECS; poll_time = poll_immediate; repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL_XON received\n"); gtmsource_restart_heartbeat; /*Macro*/ REPL_DPRINT1("Restarting HEARTBEAT\n"); xon_wait_logged = FALSE; /* In catchup mode, we do not receive as often as we would have in non catchup mode. * The consequence of this may be that we do not react to XOFF quickly enough, * making it worse for the replication pipe. We may end up with multiple XOFFs (with * intervening XONs) sitting on the replication pipe that are yet to be received * and processed by the source server. To avoid such a situation, on receipt of * an XON, we immediately force a check on the incoming pipe thereby draining * all pending XOFF/XONs, keeping the pipe smooth. Also, there is less likelihood * of missing a HEARTBEAT response (that is perhaps on the pipe) which may lead to * connection breakage although the pipe is alive and well. * * We will force a check regardless of mode (catchup or non catchup) as we may * be pounding the secondary even in non catchup mode */ force_recv_check = TRUE; break; case REPL_BADTRANS: case REPL_CMP2UNCMP: case REPL_START_JNL_SEQNO: QWASSIGN(recvd_seqno, *(seq_num *)>msource_msgp->msg[0]); if (msg_is_cross_endian) recvd_seqno = GTM_BYTESWAP_64(recvd_seqno); gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_SEARCHING_FOR_RESTART; if ((REPL_BADTRANS == gtmsource_msgp->type) || (REPL_CMP2UNCMP == gtmsource_msgp->type)) { already_communicated = TRUE; recvd_start_flags = START_FLAG_NONE; if (REPL_BADTRANS == gtmsource_msgp->type) repl_log(gtmsource_log_fp, TRUE, TRUE, "Received REPL_BADTRANS " "message with SEQNO %llu [0x%llx]\n", recvd_seqno, recvd_seqno); else { repl_log(gtmsource_log_fp, TRUE, TRUE, "Received REPL_CMP2UNCMP " "message with SEQNO %llu [0x%llx]\n", recvd_seqno, recvd_seqno); repl_log(gtmsource_log_fp, TRUE, FALSE, "Defaulting to NO compression for this connection\n"); gtmsource_received_cmp2uncmp_msg = TRUE; } } else { recvd_start_flags = ((repl_start_msg_ptr_t)gtmsource_msgp)->start_flags; if (msg_is_cross_endian) recvd_start_flags = GTM_BYTESWAP_32(recvd_start_flags); already_communicated = FALSE; repl_log(gtmsource_log_fp, TRUE, TRUE, "Received REPL_START_JNL_SEQNO message with SEQNO %llu [0x%llx]. " "Possible crash of recvr/update process\n", recvd_seqno, recvd_seqno); } break; case REPL_HEARTBEAT: if (msg_is_cross_endian) { heartbeat_msg = (repl_heartbeat_msg_ptr_t)gtmsource_msgp; tmp_seqno = *(seq_num *)&heartbeat_msg->ack_seqno[0]; tmp_seqno = GTM_BYTESWAP_64(tmp_seqno); *(seq_num *)&heartbeat_msg->ack_seqno[0] = tmp_seqno; tmp_time4 = *(gtm_time4_t *)&heartbeat_msg->ack_time[0]; tmp_time4 = GTM_BYTESWAP_32(tmp_time4); *(gtm_time4_t *)&heartbeat_msg->ack_time[0] = tmp_time4; } gtmsource_process_heartbeat((repl_heartbeat_msg_ptr_t)gtmsource_msgp); break; default: repl_log(gtmsource_log_fp, TRUE, TRUE, "Message of unknown type %d of length %d " "bytes received; hex dump follows\n", gtmsource_msgp->type, recvd_len); for (index = 0; index < MIN(recvd_len, gtmsource_msgbufsiz - REPL_MSG_HDRLEN); ) { repl_log(gtmsource_log_fp, FALSE, FALSE, "%.2x ", gtmsource_msgp->msg[index]); if ((++index) % MAX_HEXDUMP_CHARS_PER_LINE == 0) repl_log(gtmsource_log_fp, FALSE, TRUE, "\n"); } repl_log(gtmsource_log_fp, FALSE, TRUE, "\n"); /* flush BEFORE the assert */ assert(FALSE); break; } } else if (SS_NORMAL != status) { if (EREPL_RECV == repl_errno) { if (REPL_CONN_RESET(status)) { /* Connection reset */ repl_log(gtmsource_log_fp, TRUE, TRUE, "Connection reset while attempting to receive from secondary." " Status = %d ; %s\n", status, STRERROR(status)); repl_close(>msource_sock_fd); SHORT_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_CLOSE_CONN); gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION; break; } else { SNPRINTF(err_string, SIZEOF(err_string), "Error receiving Control message from Receiver. Error in recv : %s", STRERROR(status)); rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_STRING(err_string)); } } else if (EREPL_SELECT == repl_errno) { SNPRINTF(err_string, SIZEOF(err_string), "Error receiving Control message from Receiver. Error in select : %s", STRERROR(status)); rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_STRING(err_string)); } } if (GTMSOURCE_WAITING_FOR_XON == gtmsource_state) { if (!xon_wait_logged) { repl_log(gtmsource_log_fp, TRUE, TRUE, "Waiting to receive XON\n"); gtmsource_stall_heartbeat; /* Macro */ REPL_DPRINT1("Stalling HEARTBEAT\n"); xon_wait_logged = TRUE; } if (GTMSOURCE_FH_FLUSH_INTERVAL <= difftime(gtmsource_now, gtmsource_last_flush_time)) { gtmsource_flush_fh(gtmsource_local->read_jnl_seqno); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) break; /* the outerloop will continue */ } continue; } if (GTMSOURCE_SEARCHING_FOR_RESTART == gtmsource_state || GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) { xon_wait_logged = FALSE; break; } assert(gtmsource_state == GTMSOURCE_SENDING_JNLRECS); if (force_recv_check) /* we want to poll the incoming pipe for possible XOFF */ continue; assert(pre_read_seqno == gtmsource_local->read_jnl_seqno); GRAB_GTMSOURCE_SRV_LATCH(gtmsource_local, max_epoch_interval); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) break; /* the outerloop will continue */ tot_tr_len = gtmsource_get_jnlrecs(>msource_msgp->msg[0], &data_len, gtmsource_msgbufsiz - REPL_MSG_HDRLEN, !(gtmsource_filter & EXTERNAL_FILTER)); rel_gtmsource_srv_latch(>msource_local->gtmsource_srv_latch); /* It is safe to send the journal records as we are guaranteed NO online rollback happened in between * and so we won't be sending garbage */ if (GTMSOURCE_CHANGING_MODE == gtmsource_state) return (SS_NORMAL); if (GTMSOURCE_WAITING_FOR_CONNECTION == gtmsource_state) break; if (GTMSOURCE_SEND_NEW_HISTINFO == gtmsource_state) { /* This is a signal from "gtmsource_get_jnlrecs" to send a REPL_HISTREC message first * before sending any more seqnos across. Set "gtmsource_local->send_new_histrec" to TRUE. */ assert(0 == tot_tr_len); gtmsource_local->send_new_histrec = TRUE; /* Will cause a new histinfo record to be sent first */ gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_SENDING_JNLRECS; continue; /* Send a REPL_HISTREC message first and then send journal records */ } post_read_seqno = gtmsource_local->read_jnl_seqno; if (0 <= tot_tr_len) { if (0 < data_len) { APPLY_EXT_FILTER_IF_NEEDED(gtmsource_filter, gtmsource_msgp, data_len, tot_tr_len); gtmsource_msgp->type = REPL_TR_JNL_RECS; gtmsource_msgp->len = data_len + REPL_MSG_HDRLEN; send_msgp = gtmsource_msgp; send_tr_len = tot_tr_len; intfilter_error = FALSE; if (gtmsource_filter & INTERNAL_FILTER) { in_buff = gtmsource_msgp->msg; in_buflen = data_len; /* size of the first journal record in the converted buffer */ out_buffmsg = repl_filter_buff; out_buff = out_buffmsg + REPL_MSG_HDRLEN; out_bufsiz = repl_filter_bufsiz - REPL_MSG_HDRLEN; remaining_len = tot_tr_len; while (JREC_PREFIX_SIZE <= remaining_len) { filter_seqno = ((struct_jrec_null *)(in_buff))->jnl_seqno; DEBUG_ONLY( save_inbuff = in_buff; save_outbuff = out_buff; ) APPLY_INT_FILTER(in_buff, in_buflen, out_buff, out_buflen, out_bufsiz, status); /* Internal filters should not modify the incoming pointers. Assert that. */ assert((save_inbuff == in_buff) && (save_outbuff == out_buff)); if (SS_NORMAL == status) { /* adjust various pointers and book-keeping values to move to next * record. */ ((repl_msg_ptr_t)(out_buffmsg))->type = REPL_TR_JNL_RECS; ((repl_msg_ptr_t)(out_buffmsg))->len = out_buflen + REPL_MSG_HDRLEN; out_buffmsg = (out_buff + out_buflen); remaining_len -= (in_buflen + REPL_MSG_HDRLEN); assert(0 <= remaining_len); if (0 >= remaining_len) break; in_buff += in_buflen; in_buflen = ((repl_msg_ptr_t)(in_buff))->len - REPL_MSG_HDRLEN; in_buff += REPL_MSG_HDRLEN; out_buff = (out_buffmsg + REPL_MSG_HDRLEN); out_bufsiz -= (out_buflen + REPL_MSG_HDRLEN); assert(0 <= (int)out_bufsiz); } else if (EREPL_INTLFILTER_NOSPC == repl_errno) { REALLOCATE_INT_FILTER_BUFF(out_buff, out_buffmsg, out_bufsiz); /* note that in_buff and in_buflen is not changed so that we can * start from where we left */ } else /* fatal error from the internal filter */ { intfilter_error = TRUE; break; } } assert((0 == remaining_len) || intfilter_error); GTMTRIG_ONLY(ISSUE_TRIG2NOTRIG_IF_NEEDED;) send_msgp = (repl_msg_ptr_t)repl_filter_buff; send_tr_len = out_buffmsg - repl_filter_buff; if (0 == send_tr_len) { /* This is possible ONLY if the first transaction in the buffer read from * journal pool or disk encountered error while doing internal filter * conversion. Issue rts_error right away as there is nothing much we can * do at this point. */ assert(intfilter_error); assert((EREPL_INTLFILTER_BADREC == repl_errno) || (EREPL_INTLFILTER_REPLGBL2LONG == repl_errno) || (EREPL_INTLFILTER_SECNODZTRIGINTP == repl_errno) || (EREPL_INTLFILTER_MULTILINEXECUTE == repl_errno)); assert(filter_seqno == pre_read_seqno); INT_FILTER_RTS_ERROR(filter_seqno); /* no return */ } } assert(send_tr_len && (0 == (send_tr_len % REPL_MSG_ALIGN))); /* ensure that the head of the buffer has the correct type and len */ assert((REPL_TR_JNL_RECS == send_msgp->type) && (0 == (send_msgp->len % JNL_REC_START_BNDRY))); /* At this point send_msgp is the buffer to be sent and send_tr_len is the send size */ assert(remote_side->endianness_known); if (remote_side->cross_endian && (this_side->jnl_ver < remote_side->jnl_ver)) { /* Cross-endian replication with GT.M version on primary being lesser than that * the secondary. Do the endian conversion in the primary so that the secondary * can consume it as-is. * No return if the below call to repl_tr_endian_convert fails. */ repl_tr_endian_convert(send_msgp, send_tr_len, pre_read_seqno); } pre_cmpmsglen = send_tr_len; /* send_tr_len will be updated below */ if (ZLIB_CMPLVL_NONE != repl_zlib_cmp_level) { /* Compress the journal records before replicating them across the pipe. * Depending on whether the total data length to be sent is within a threshold * or not (see repl_msg.h before REPL_TR_CMP_THRESHOLD #define for why), send * either a REPL_TR_CMP_JNL_RECS or REPL_TR_CMP_JNL_RECS2 message */ msghdrlen = (REPL_TR_CMP_THRESHOLD > send_tr_len) ? REPL_MSG_HDRLEN : REPL_MSG_HDRLEN2; cmpbuflen = gtmsource_cmpmsgbufsiz - msghdrlen; cmpbufptr = ((Bytef *)gtmsource_cmpmsgp) + msghdrlen; ZLIB_COMPRESS(cmpbufptr, cmpbuflen, send_msgp, send_tr_len, repl_zlib_cmp_level, cmpret); BREAK_IF_CMP_ERROR(cmpret, send_tr_len); /* Note: break stmt. inside the macro */ if (Z_OK == cmpret) { /* Send compressed buffer */ send_msgp = gtmsource_cmpmsgp; if (REPL_TR_CMP_THRESHOLD > send_tr_len) { /* Send REPL_TR_CMP_JNL_RECS message with 8-byte header */ SET_8BYTE_CMP_MSGHDR(send_msgp, send_tr_len, cmpbuflen, msghdrlen); } else { /* Send REPL_TR_CMP_JNL_RECS2 message with 16-byte header */ SET_16BYTE_CMP_MSGHDR(send_msgp, send_tr_len, cmpbuflen, msghdrlen); } } else { /* Send normal buffer */ repl_log(gtmsource_log_fp, TRUE, FALSE, "Defaulting to NO compression\n"); repl_zlib_cmp_level = ZLIB_CMPLVL_NONE; /* no compression */ gtmsource_local->repl_zlib_cmp_level = repl_zlib_cmp_level; } } assert((send_tr_len == pre_cmpmsglen) || (ZLIB_CMPLVL_NONE != repl_zlib_cmp_level)); assert(0 == (send_tr_len % REPL_MSG_ALIGN)); /* The following loop tries to send multiple seqnos in one shot. resync_seqno gets * updated once the send is completely successful. If an error occurs in the middle * of the send, it is possible that we successfully sent a few seqnos to the other side. * In this case resync_seqno should be updated to reflect those seqnos. Not doing so * might cause the secondary to get ahead of the primary in terms of resync_seqno. * Although it is possible to determine the exact seqno where the send partially failed, * we update resync_seqno as if all seqnos were successfully sent (It is ok for the * resync_seqno on the primary side to be a little more than the actual value as long as * the secondary side has an accurate value of resync_seqno. This is because the * resync_seqno of the system is the minimum of the resync_seqno of both primary * and secondary). This is done by the call to gtmsource_flush_fh() done within the * REPL_SEND_LOOP macro as well as in the (SS_NORMAL != status) if condition below. * Note that all of this is applicable only in a dualsite replication scenario. In * case of a multisite scenario, it is always the receiver server that tells the * sequence number from where the source server should start sending. So, even if * the source server notes down a higher value of journal sequence number in * jnlpool.gtmsource_local->read_jnl_seqno, it is not a problem since the receiver * server will communicate the appropriate sequence number as part of the histinfo * exchange. */ REPL_SEND_LOOP(gtmsource_sock_fd, send_msgp, send_tr_len, catchup, &poll_immediate) { gtmsource_poll_actions(FALSE); if (GTMSOURCE_CHANGING_MODE == gtmsource_state) { gtmsource_flush_fh(post_read_seqno); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) break; return (SS_NORMAL); } } if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) break; /* the outerloop will continue */ if (SS_NORMAL != status) { gtmsource_flush_fh(post_read_seqno); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) break; /* the outerloop will continue */ if (REPL_CONN_RESET(status) && EREPL_SEND == repl_errno) { repl_log(gtmsource_log_fp, TRUE, TRUE, "Connection reset while sending seqno data from " "%llu [0x%llx] to %llu [0x%llx]. Status = %d ; %s\n", pre_read_seqno, pre_read_seqno, post_read_seqno, post_read_seqno, status, STRERROR(status)); repl_close(>msource_sock_fd); SHORT_SLEEP(GTMSOURCE_WAIT_FOR_RECEIVER_CLOSE_CONN); gtmsource_state = gtmsource_local->gtmsource_state = GTMSOURCE_WAITING_FOR_CONNECTION; break; } if (EREPL_SEND == repl_errno) { SNPRINTF(err_string, SIZEOF(err_string), "Error sending DATA. Error in send : %s", STRERROR(status)); rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_STRING(err_string)); } if (EREPL_SELECT == repl_errno) { SNPRINTF(err_string, SIZEOF(err_string), "Error sending DATA. Error in select : %s", STRERROR(status)); rts_error(VARLSTCNT(6) ERR_REPLCOMM, 0, ERR_TEXT, 2, RTS_ERROR_STRING(err_string)); } } if (intfilter_error) { /* Now that we are done sending whatever buffer was filter converted, issue * the error. This will bring down the source server (due to the rts_error). * At this point, jnlpool.gtmsource_local->read_jnl_seqno could effectively * be behind the receiver server's journal sequence number. But, that is * okay since as part of reconnection (when the source server comes back up), * the receiever server will communicate the appropriate sequence number as * part of the histinfo exchange. */ assert((EREPL_INTLFILTER_BADREC == repl_errno) || (EREPL_INTLFILTER_REPLGBL2LONG == repl_errno) || (EREPL_INTLFILTER_SECNODZTRIGINTP == repl_errno) || (EREPL_INTLFILTER_MULTILINEXECUTE == repl_errno)); assert(filter_seqno <= post_read_seqno); INT_FILTER_RTS_ERROR(filter_seqno); /* no return */ } jnlpool.gtmsource_local->read_jnl_seqno = post_read_seqno; if (GTMSOURCE_FH_FLUSH_INTERVAL <= difftime(gtmsource_now, gtmsource_last_flush_time)) { gtmsource_flush_fh(post_read_seqno); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) break; /* the outerloop will continue */ } repl_source_cmp_sent += (qw_num)send_tr_len; repl_source_msg_sent += (qw_num)pre_cmpmsglen; repl_source_data_sent += (qw_num)(pre_cmpmsglen) - (post_read_seqno - pre_read_seqno) * REPL_MSG_HDRLEN; log_seqno = post_read_seqno - 1; /* post_read_seqno is the "next" seqno to be sent, * not the last one we sent */ if (gtmsource_logstats || (log_seqno - lastlog_seqno >= log_interval)) { /* print always when STATSLOG is ON, or when the log interval has passed */ trans_sent_cnt += (log_seqno - lastlog_seqno); /* jctl->jnl_seqno >= post_read_seqno is the most common case; * see gtmsource_readpool() for when the rare case can occur */ jnl_seqno = jctl->jnl_seqno; assert(jnl_seqno >= post_read_seqno - 1); diff_seqno = (jnl_seqno >= post_read_seqno) ? (jnl_seqno - post_read_seqno) : 0; repl_log(gtmsource_log_fp, FALSE, FALSE, "REPL INFO - Seqno : %llu [0x%llx]", log_seqno, log_seqno); repl_log(gtmsource_log_fp, FALSE, FALSE, " Jnl Total : %llu [0x%llx] Msg Total :" " %llu [0x%llx] CmpMsg Total : %llu [0x%llx] ", repl_source_data_sent, repl_source_data_sent, repl_source_msg_sent, repl_source_msg_sent, repl_source_cmp_sent, repl_source_cmp_sent); repl_log(gtmsource_log_fp, FALSE, TRUE, "Current backlog : %llu [0x%llx]\n", diff_seqno, diff_seqno); /* gtmsource_now is updated by the heartbeat protocol every heartbeat * interval. To cut down on calls to time(), we use gtmsource_now as the * time to figure out if we have to log statistics. This works well as the * logging interval generally is larger than the heartbeat interval, and that * the heartbeat protocol is running when we are sending data. The consequence * although is that we may defer logging when we would have logged. We can live * with that given the benefit of not calling time related system calls. * Currently, the logging interval is not changeable by users. When/if we provide * means of choosing log interval, this code may have to be re-examined. * Vinaya 2003, Sep 08 */ assert(0 != gtmsource_now); /* must hold if we are sending data */ repl_source_this_log_time = gtmsource_now; /* approximate time, in the worst case, * behind by heartbeat interval */ assert(repl_source_this_log_time >= repl_source_prev_log_time); time_elapsed = difftime(repl_source_this_log_time, repl_source_prev_log_time); if ((double)GTMSOURCE_LOGSTATS_INTERVAL <= time_elapsed) { delta_sent_cnt = trans_sent_cnt - last_log_tr_sent_cnt; delta_data_sent = repl_source_data_sent - repl_source_lastlog_data_sent; delta_msg_sent = repl_source_msg_sent - repl_source_lastlog_msg_sent; repl_log(gtmsource_log_fp, TRUE, FALSE, "REPL INFO since last log : " "Time elapsed : %00.f Tr sent : %llu [0x%llx] Tr bytes : %llu " "[0x%llx] Msg bytes : %llu [0x%llx]\n", time_elapsed, delta_sent_cnt, delta_sent_cnt, delta_data_sent, delta_data_sent, delta_msg_sent, delta_msg_sent); repl_log(gtmsource_log_fp, TRUE, TRUE, "REPL INFO since last log : " "Time elapsed : %00.f Tr sent/s : %f Tr bytes/s : %f " "Msg bytes/s : %f\n", time_elapsed, (float)delta_sent_cnt / time_elapsed, (float)delta_data_sent / time_elapsed, (float)delta_msg_sent / time_elapsed); repl_source_lastlog_data_sent = repl_source_data_sent; repl_source_lastlog_msg_sent = repl_source_msg_sent; last_log_tr_sent_cnt = trans_sent_cnt; repl_source_prev_log_time = repl_source_this_log_time; } lastlog_seqno = log_seqno; } poll_time = poll_immediate; } else /* data_len == 0 */ { /* nothing to send */ gtmsource_flush_fh(post_read_seqno); if (GTMSOURCE_HANDLE_ONLN_RLBK == gtmsource_state) break; /* the outerloop will continue */ poll_time = poll_wait; rel_quant(); /* give up processor and let other processes run */ } } else /* else tot_tr_len < 0, error */ { if (0 < data_len) /* Insufficient buffer space, increase the buffer space */ gtmsource_alloc_msgbuff(data_len + REPL_MSG_HDRLEN); else GTMASSERT; /* Major problems */ } } } }