From 8528248fc1b400f533f576d42556ec9025f9056c Mon Sep 17 00:00:00 2001 From: Greig Wise Date: Wed, 23 Jan 2019 17:52:52 -0500 Subject: [PATCH 1/2] Add support to include a list of column names for updates in the case that toasted data for those columns is missing --- wal2json.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/wal2json.c b/wal2json.c index 53a933f..f1b75ce 100644 --- a/wal2json.c +++ b/wal2json.c @@ -39,6 +39,7 @@ typedef struct bool include_type_oids; /* include data type oids */ bool include_typmod; /* include typmod in types */ bool include_not_null; /* include not-null constraints */ + bool include_missing_toast; /* include list of missing toast columns */ bool pretty_print; /* pretty-print JSON? */ bool write_in_chunks; /* write in chunks? */ @@ -139,6 +140,7 @@ pg_decode_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, bool is data->write_in_chunks = false; data->include_lsn = false; data->include_not_null = false; + data->include_missing_toast = false; data->filter_tables = NIL; /* pretty print */ @@ -359,6 +361,19 @@ pg_decode_startup(LogicalDecodingContext *ctx, OutputPluginOptions *opt, bool is pfree(rawstr); } } + else if (strcmp(elem->defname, "include-missing-toast") == 0) + { + if (elem->arg == NULL) + { + elog(DEBUG1, "include-missing-toast argument is null"); + data->include_missing_toast = true; + } + else if (!parse_bool(strVal(elem->arg), &data->include_missing_toast)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("could not parse value \"%s\" for parameter \"%s\"", + strVal(elem->arg), elem->defname))); + } else { ereport(ERROR, @@ -455,11 +470,13 @@ tuple_to_stringinfo(LogicalDecodingContext *ctx, TupleDesc tupdesc, HeapTuple tu int natt; StringInfoData colnames; + StringInfoData missingtoastcols; StringInfoData coltypes; StringInfoData coltypeoids; StringInfoData colnotnulls; StringInfoData colvalues; char comma[3] = ""; + char toastcomma[3] = ""; data = ctx->output_plugin_private; @@ -469,6 +486,8 @@ tuple_to_stringinfo(LogicalDecodingContext *ctx, TupleDesc tupdesc, HeapTuple tu initStringInfo(&coltypeoids); if (data->include_not_null) initStringInfo(&colnotnulls); + if (data->include_missing_toast) + initStringInfo(&missingtoastcols); initStringInfo(&colvalues); /* @@ -493,6 +512,8 @@ tuple_to_stringinfo(LogicalDecodingContext *ctx, TupleDesc tupdesc, HeapTuple tu appendStringInfo(&coltypeoids, "%s%s%s\"columntypeoids\":%s[", data->ht, data->ht, data->ht, data->sp); if (data->include_not_null) appendStringInfo(&colnotnulls, "%s%s%s\"columnoptionals\":%s[", data->ht, data->ht, data->ht, data->sp); + if (data->include_missing_toast) + appendStringInfo(&missingtoastcols, "%s%s%s\"missingtoastcols\":%s[", data->ht, data->ht, data->ht, data->sp); appendStringInfo(&colvalues, "%s%s%s\"columnvalues\":%s[", data->ht, data->ht, data->ht, data->sp); } @@ -573,6 +594,16 @@ tuple_to_stringinfo(LogicalDecodingContext *ctx, TupleDesc tupdesc, HeapTuple tu /* XXX Unchanged TOAST Datum does not need to be output */ if (!isnull && typisvarlena && VARATT_IS_EXTERNAL_ONDISK(origval)) { + if (data->include_missing_toast) + { + appendStringInfo(&missingtoastcols, "%s", toastcomma); + escape_json(&missingtoastcols, NameStr(attr->attname)); + } + + /* first missing toast has no comma */ + if (strcmp(toastcomma, "") == 0) + snprintf(toastcomma, 3, ",%s", data->sp); + elog(DEBUG1, "column \"%s\" has an unchanged TOAST", NameStr(attr->attname)); continue; } @@ -699,6 +730,8 @@ tuple_to_stringinfo(LogicalDecodingContext *ctx, TupleDesc tupdesc, HeapTuple tu appendStringInfo(&coltypeoids, "],%s", data->nl); if (data->include_not_null) appendStringInfo(&colnotnulls, "],%s", data->nl); + if (data->include_missing_toast) + appendStringInfo(&missingtoastcols, "],%s", data->nl); if (hasreplident) appendStringInfo(&colvalues, "],%s", data->nl); else @@ -713,6 +746,8 @@ tuple_to_stringinfo(LogicalDecodingContext *ctx, TupleDesc tupdesc, HeapTuple tu appendStringInfoString(ctx->out, coltypeoids.data); if (data->include_not_null) appendStringInfoString(ctx->out, colnotnulls.data); + if (data->include_missing_toast) + appendStringInfoString(ctx->out, missingtoastcols.data); appendStringInfoString(ctx->out, colvalues.data); pfree(colnames.data); @@ -721,6 +756,8 @@ tuple_to_stringinfo(LogicalDecodingContext *ctx, TupleDesc tupdesc, HeapTuple tu pfree(coltypeoids.data); if (data->include_not_null) pfree(colnotnulls.data); + if (data->include_missing_toast) + pfree(missingtoastcols.data); pfree(colvalues.data); } From 5871a0942b806c2e422dc1f84bddd4a72dc1a1b6 Mon Sep 17 00:00:00 2001 From: Greig Wise Date: Tue, 12 Feb 2019 11:44:12 -0500 Subject: [PATCH 2/2] Add tests and update docs --- Makefile | 3 +- README.md | 1 + expected/missing_toast.out | 103 +++++++++++++++++++++++++++++++++++++ sql/missing_toast.sql | 30 +++++++++++ 4 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 expected/missing_toast.out create mode 100644 sql/missing_toast.sql diff --git a/Makefile b/Makefile index c09b4b7..8d17e5a 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,8 @@ MODULES = wal2json # message test will fail for <= 9.5 REGRESS = cmdline insert1 update1 update2 update3 update4 delete1 delete2 \ delete3 delete4 savepoint specialvalue toast bytea message typmod \ - filtertable selecttable include_timestamp include_lsn include_xids + filtertable selecttable include_timestamp include_lsn include_xids \ + missing_toast PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/README.md b/README.md index a9f6c5f..4470976 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Parameters * `filter-tables`: exclude rows from the specified tables. Default is empty which means that no table will be filtered. It is a comma separated value. The tables should be schema-qualified. `*.foo` means table foo in all schemas and `bar.*` means all tables in schema bar. Special characters (space, single quote, comma, period, asterisk) must be escaped with backslash. Schema and table are case-sensitive. Table `"public"."Foo bar"` should be specified as `public.Foo\ bar`. * `add-tables`: include only rows from the specified tables. Default is all tables from all schemas. It has the same rules from `filter-tables`. * `format-version`: defines which format to use. Default is _1_. +* `include-missing-toast`: include a list of columns which are not included in return data since they are toasted and were not updated. Default is _false_ Examples ======== diff --git a/expected/missing_toast.out b/expected/missing_toast.out new file mode 100644 index 0000000..f54f6b8 --- /dev/null +++ b/expected/missing_toast.out @@ -0,0 +1,103 @@ +\set VERBOSITY terse +-- predictability +SET synchronous_commit = on; +DROP TABLE IF EXISTS missing_toast; +NOTICE: table "missing_toast" does not exist, skipping +CREATE TABLE missing_toast ( +id serial primary key, +an_integer integer, +toasted_col1 text, +toasted_col2 text +); +SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2json'); + ?column? +---------- + init +(1 row) + +-- uncompressed external toast data +INSERT INTO missing_toast (toasted_col1, toasted_col2) SELECT string_agg(g.i::text, ''), string_agg((g.i*2)::text, '') FROM generate_series(1, 2000) g(i); +-- update of existing column +UPDATE missing_toast SET toasted_col1 = (SELECT string_agg((g.i*2)::text, '') FROM generate_series(1, 2000) g(i)) where id = 1; +UPDATE missing_toast set an_integer = 1 where id = 1; +UPDATE missing_toast SET toasted_col1 = (SELECT string_agg((g.i*2)::text, '') FROM generate_series(1, 2000) g(i)), +toasted_col2 = (SELECT string_agg((g.i*2)::text, '') FROM generate_series(1, 2000) g(i)) +where id = 1; +SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'pretty-print', '1', 'include-typmod', '0', 'include-missing-toast', '1'); + data +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + { + + "change": [ + + { + + "kind": "insert", + + "schema": "public", + + "table": "missing_toast", + + "columnnames": ["id", "an_integer", "toasted_col1", "toasted_col2"], + + "columntypes": ["int4", "int4", "text", "text"], + + "missingtoastcols": [], + + "columnvalues": [1, null, "12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000", "24681012141618202224262830323436384042444648505254565860626466687072747678808284868890929496981001021041061081101121141161181201221241261281301321341361381401421441461481501521541561581601621641661681701721741761781801821841861881901921941961982002022042062082102122142162182202222242262282302322342362382402422442462482502522542562582602622642662682702722742762782802822842862882902922942962983003023043063083103123143163183203223243263283303323343363383403423443463483503523543563583603623643663683703723743763783803823843863883903923943963984004024044064084104124144164184204224244264284304324344364384404424444464484504524544564584604624644664684704724744764784804824844864884904924944964985005025045065085105125145165185205225245265285305325345365385405425445465485505525545565585605625645665685705725745765785805825845865885905925945965986006026046066086106126146166186206226246266286306326346366386406426446466486506526546566586606626646666686706726746766786806826846866886906926946966987007027047067087107127147167187207227247267287307327347367387407427447467487507527547567587607627647667687707727747767787807827847867887907927947967988008028048068088108128148168188208228248268288308328348368388408428448468488508528548568588608628648668688708728748768788808828848868888908928948968989009029049069089109129149169189209229249269289309329349369389409429449469489509529549569589609629649669689709729749769789809829849869889909929949969981000100210041006100810101012101410161018102010221024102610281030103210341036103810401042104410461048105010521054105610581060106210641066106810701072107410761078108010821084108610881090109210941096109811001102110411061108111011121114111611181120112211241126112811301132113411361138114011421144114611481150115211541156115811601162116411661168117011721174117611781180118211841186118811901192119411961198120012021204120612081210121212141216121812201222122412261228123012321234123612381240124212441246124812501252125412561258126012621264126612681270127212741276127812801282128412861288129012921294129612981300130213041306130813101312131413161318132013221324132613281330133213341336133813401342134413461348135013521354135613581360136213641366136813701372137413761378138013821384138613881390139213941396139814001402140414061408141014121414141614181420142214241426142814301432143414361438144014421444144614481450145214541456145814601462146414661468147014721474147614781480148214841486148814901492149414961498150015021504150615081510151215141516151815201522152415261528153015321534153615381540154215441546154815501552155415561558156015621564156615681570157215741576157815801582158415861588159015921594159615981600160216041606160816101612161416161618162016221624162616281630163216341636163816401642164416461648165016521654165616581660166216641666166816701672167416761678168016821684168616881690169216941696169817001702170417061708171017121714171617181720172217241726172817301732173417361738174017421744174617481750175217541756175817601762176417661768177017721774177617781780178217841786178817901792179417961798180018021804180618081810181218141816181818201822182418261828183018321834183618381840184218441846184818501852185418561858186018621864186618681870187218741876187818801882188418861888189018921894189618981900190219041906190819101912191419161918192019221924192619281930193219341936193819401942194419461948195019521954195619581960196219641966196819701972197419761978198019821984198619881990199219941996199820002002200420062008201020122014201620182020202220242026202820302032203420362038204020422044204620482050205220542056205820602062206420662068207020722074207620782080208220842086208820902092209420962098210021022104210621082110211221142116211821202122212421262128213021322134213621382140214221442146214821502152215421562158216021622164216621682170217221742176217821802182218421862188219021922194219621982200220222042206220822102212221422162218222022222224222622282230223222342236223822402242224422462248225022522254225622582260226222642266226822702272227422762278228022822284228622882290229222942296229823002302230423062308231023122314231623182320232223242326232823302332233423362338234023422344234623482350235223542356235823602362236423662368237023722374237623782380238223842386238823902392239423962398240024022404240624082410241224142416241824202422242424262428243024322434243624382440244224442446244824502452245424562458246024622464246624682470247224742476247824802482248424862488249024922494249624982500250225042506250825102512251425162518252025222524252625282530253225342536253825402542254425462548255025522554255625582560256225642566256825702572257425762578258025822584258625882590259225942596259826002602260426062608261026122614261626182620262226242626262826302632263426362638264026422644264626482650265226542656265826602662266426662668267026722674267626782680268226842686268826902692269426962698270027022704270627082710271227142716271827202722272427262728273027322734273627382740274227442746274827502752275427562758276027622764276627682770277227742776277827802782278427862788279027922794279627982800280228042806280828102812281428162818282028222824282628282830283228342836283828402842284428462848285028522854285628582860286228642866286828702872287428762878288028822884288628882890289228942896289829002902290429062908291029122914291629182920292229242926292829302932293429362938294029422944294629482950295229542956295829602962296429662968297029722974297629782980298229842986298829902992299429962998300030023004300630083010301230143016301830203022302430263028303030323034303630383040304230443046304830503052305430563058306030623064306630683070307230743076307830803082308430863088309030923094309630983100310231043106310831103112311431163118312031223124312631283130313231343136313831403142314431463148315031523154315631583160316231643166316831703172317431763178318031823184318631883190319231943196319832003202320432063208321032123214321632183220322232243226322832303232323432363238324032423244324632483250325232543256325832603262326432663268327032723274327632783280328232843286328832903292329432963298330033023304330633083310331233143316331833203322332433263328333033323334333633383340334233443346334833503352335433563358336033623364336633683370337233743376337833803382338433863388339033923394339633983400340234043406340834103412341434163418342034223424342634283430343234343436343834403442344434463448345034523454345634583460346234643466346834703472347434763478348034823484348634883490349234943496349835003502350435063508351035123514351635183520352235243526352835303532353435363538354035423544354635483550355235543556355835603562356435663568357035723574357635783580358235843586358835903592359435963598360036023604360636083610361236143616361836203622362436263628363036323634363636383640364236443646364836503652365436563658366036623664366636683670367236743676367836803682368436863688369036923694369636983700370237043706370837103712371437163718372037223724372637283730373237343736373837403742374437463748375037523754375637583760376237643766376837703772377437763778378037823784378637883790379237943796379838003802380438063808381038123814381638183820382238243826382838303832383438363838384038423844384638483850385238543856385838603862386438663868387038723874387638783880388238843886388838903892389438963898390039023904390639083910391239143916391839203922392439263928393039323934393639383940394239443946394839503952395439563958396039623964396639683970397239743976397839803982398439863988399039923994399639984000"] + + } + + ] + + } + { + + "change": [ + + { + + "kind": "update", + + "schema": "public", + + "table": "missing_toast", + + "columnnames": ["id", "an_integer", "toasted_col1"], + + "columntypes": ["int4", "int4", "text"], + + "missingtoastcols": ["toasted_col2"], + + "columnvalues": [1, null, "24681012141618202224262830323436384042444648505254565860626466687072747678808284868890929496981001021041061081101121141161181201221241261281301321341361381401421441461481501521541561581601621641661681701721741761781801821841861881901921941961982002022042062082102122142162182202222242262282302322342362382402422442462482502522542562582602622642662682702722742762782802822842862882902922942962983003023043063083103123143163183203223243263283303323343363383403423443463483503523543563583603623643663683703723743763783803823843863883903923943963984004024044064084104124144164184204224244264284304324344364384404424444464484504524544564584604624644664684704724744764784804824844864884904924944964985005025045065085105125145165185205225245265285305325345365385405425445465485505525545565585605625645665685705725745765785805825845865885905925945965986006026046066086106126146166186206226246266286306326346366386406426446466486506526546566586606626646666686706726746766786806826846866886906926946966987007027047067087107127147167187207227247267287307327347367387407427447467487507527547567587607627647667687707727747767787807827847867887907927947967988008028048068088108128148168188208228248268288308328348368388408428448468488508528548568588608628648668688708728748768788808828848868888908928948968989009029049069089109129149169189209229249269289309329349369389409429449469489509529549569589609629649669689709729749769789809829849869889909929949969981000100210041006100810101012101410161018102010221024102610281030103210341036103810401042104410461048105010521054105610581060106210641066106810701072107410761078108010821084108610881090109210941096109811001102110411061108111011121114111611181120112211241126112811301132113411361138114011421144114611481150115211541156115811601162116411661168117011721174117611781180118211841186118811901192119411961198120012021204120612081210121212141216121812201222122412261228123012321234123612381240124212441246124812501252125412561258126012621264126612681270127212741276127812801282128412861288129012921294129612981300130213041306130813101312131413161318132013221324132613281330133213341336133813401342134413461348135013521354135613581360136213641366136813701372137413761378138013821384138613881390139213941396139814001402140414061408141014121414141614181420142214241426142814301432143414361438144014421444144614481450145214541456145814601462146414661468147014721474147614781480148214841486148814901492149414961498150015021504150615081510151215141516151815201522152415261528153015321534153615381540154215441546154815501552155415561558156015621564156615681570157215741576157815801582158415861588159015921594159615981600160216041606160816101612161416161618162016221624162616281630163216341636163816401642164416461648165016521654165616581660166216641666166816701672167416761678168016821684168616881690169216941696169817001702170417061708171017121714171617181720172217241726172817301732173417361738174017421744174617481750175217541756175817601762176417661768177017721774177617781780178217841786178817901792179417961798180018021804180618081810181218141816181818201822182418261828183018321834183618381840184218441846184818501852185418561858186018621864186618681870187218741876187818801882188418861888189018921894189618981900190219041906190819101912191419161918192019221924192619281930193219341936193819401942194419461948195019521954195619581960196219641966196819701972197419761978198019821984198619881990199219941996199820002002200420062008201020122014201620182020202220242026202820302032203420362038204020422044204620482050205220542056205820602062206420662068207020722074207620782080208220842086208820902092209420962098210021022104210621082110211221142116211821202122212421262128213021322134213621382140214221442146214821502152215421562158216021622164216621682170217221742176217821802182218421862188219021922194219621982200220222042206220822102212221422162218222022222224222622282230223222342236223822402242224422462248225022522254225622582260226222642266226822702272227422762278228022822284228622882290229222942296229823002302230423062308231023122314231623182320232223242326232823302332233423362338234023422344234623482350235223542356235823602362236423662368237023722374237623782380238223842386238823902392239423962398240024022404240624082410241224142416241824202422242424262428243024322434243624382440244224442446244824502452245424562458246024622464246624682470247224742476247824802482248424862488249024922494249624982500250225042506250825102512251425162518252025222524252625282530253225342536253825402542254425462548255025522554255625582560256225642566256825702572257425762578258025822584258625882590259225942596259826002602260426062608261026122614261626182620262226242626262826302632263426362638264026422644264626482650265226542656265826602662266426662668267026722674267626782680268226842686268826902692269426962698270027022704270627082710271227142716271827202722272427262728273027322734273627382740274227442746274827502752275427562758276027622764276627682770277227742776277827802782278427862788279027922794279627982800280228042806280828102812281428162818282028222824282628282830283228342836283828402842284428462848285028522854285628582860286228642866286828702872287428762878288028822884288628882890289228942896289829002902290429062908291029122914291629182920292229242926292829302932293429362938294029422944294629482950295229542956295829602962296429662968297029722974297629782980298229842986298829902992299429962998300030023004300630083010301230143016301830203022302430263028303030323034303630383040304230443046304830503052305430563058306030623064306630683070307230743076307830803082308430863088309030923094309630983100310231043106310831103112311431163118312031223124312631283130313231343136313831403142314431463148315031523154315631583160316231643166316831703172317431763178318031823184318631883190319231943196319832003202320432063208321032123214321632183220322232243226322832303232323432363238324032423244324632483250325232543256325832603262326432663268327032723274327632783280328232843286328832903292329432963298330033023304330633083310331233143316331833203322332433263328333033323334333633383340334233443346334833503352335433563358336033623364336633683370337233743376337833803382338433863388339033923394339633983400340234043406340834103412341434163418342034223424342634283430343234343436343834403442344434463448345034523454345634583460346234643466346834703472347434763478348034823484348634883490349234943496349835003502350435063508351035123514351635183520352235243526352835303532353435363538354035423544354635483550355235543556355835603562356435663568357035723574357635783580358235843586358835903592359435963598360036023604360636083610361236143616361836203622362436263628363036323634363636383640364236443646364836503652365436563658366036623664366636683670367236743676367836803682368436863688369036923694369636983700370237043706370837103712371437163718372037223724372637283730373237343736373837403742374437463748375037523754375637583760376237643766376837703772377437763778378037823784378637883790379237943796379838003802380438063808381038123814381638183820382238243826382838303832383438363838384038423844384638483850385238543856385838603862386438663868387038723874387638783880388238843886388838903892389438963898390039023904390639083910391239143916391839203922392439263928393039323934393639383940394239443946394839503952395439563958396039623964396639683970397239743976397839803982398439863988399039923994399639984000"], + + "oldkeys": { + + "keynames": ["id"], + + "keytypes": ["int4"], + + "keyvalues": [1] + + } + + } + + ] + + } + { + + "change": [ + + { + + "kind": "update", + + "schema": "public", + + "table": "missing_toast", + + "columnnames": ["id", "an_integer"], + + "columntypes": ["int4", "int4"], + + "missingtoastcols": ["toasted_col1", "toasted_col2"], + + "columnvalues": [1, 1], + + "oldkeys": { + + "keynames": ["id"], + + "keytypes": ["int4"], + + "keyvalues": [1] + + } + + } + + ] + + } + { + + "change": [ + + { + + "kind": "update", + + "schema": "public", + + "table": "missing_toast", + + "columnnames": ["id", "an_integer", "toasted_col1", "toasted_col2"], + + "columntypes": ["int4", "int4", "text", "text"], + + "missingtoastcols": [], + + "columnvalues": [1, 1, "24681012141618202224262830323436384042444648505254565860626466687072747678808284868890929496981001021041061081101121141161181201221241261281301321341361381401421441461481501521541561581601621641661681701721741761781801821841861881901921941961982002022042062082102122142162182202222242262282302322342362382402422442462482502522542562582602622642662682702722742762782802822842862882902922942962983003023043063083103123143163183203223243263283303323343363383403423443463483503523543563583603623643663683703723743763783803823843863883903923943963984004024044064084104124144164184204224244264284304324344364384404424444464484504524544564584604624644664684704724744764784804824844864884904924944964985005025045065085105125145165185205225245265285305325345365385405425445465485505525545565585605625645665685705725745765785805825845865885905925945965986006026046066086106126146166186206226246266286306326346366386406426446466486506526546566586606626646666686706726746766786806826846866886906926946966987007027047067087107127147167187207227247267287307327347367387407427447467487507527547567587607627647667687707727747767787807827847867887907927947967988008028048068088108128148168188208228248268288308328348368388408428448468488508528548568588608628648668688708728748768788808828848868888908928948968989009029049069089109129149169189209229249269289309329349369389409429449469489509529549569589609629649669689709729749769789809829849869889909929949969981000100210041006100810101012101410161018102010221024102610281030103210341036103810401042104410461048105010521054105610581060106210641066106810701072107410761078108010821084108610881090109210941096109811001102110411061108111011121114111611181120112211241126112811301132113411361138114011421144114611481150115211541156115811601162116411661168117011721174117611781180118211841186118811901192119411961198120012021204120612081210121212141216121812201222122412261228123012321234123612381240124212441246124812501252125412561258126012621264126612681270127212741276127812801282128412861288129012921294129612981300130213041306130813101312131413161318132013221324132613281330133213341336133813401342134413461348135013521354135613581360136213641366136813701372137413761378138013821384138613881390139213941396139814001402140414061408141014121414141614181420142214241426142814301432143414361438144014421444144614481450145214541456145814601462146414661468147014721474147614781480148214841486148814901492149414961498150015021504150615081510151215141516151815201522152415261528153015321534153615381540154215441546154815501552155415561558156015621564156615681570157215741576157815801582158415861588159015921594159615981600160216041606160816101612161416161618162016221624162616281630163216341636163816401642164416461648165016521654165616581660166216641666166816701672167416761678168016821684168616881690169216941696169817001702170417061708171017121714171617181720172217241726172817301732173417361738174017421744174617481750175217541756175817601762176417661768177017721774177617781780178217841786178817901792179417961798180018021804180618081810181218141816181818201822182418261828183018321834183618381840184218441846184818501852185418561858186018621864186618681870187218741876187818801882188418861888189018921894189618981900190219041906190819101912191419161918192019221924192619281930193219341936193819401942194419461948195019521954195619581960196219641966196819701972197419761978198019821984198619881990199219941996199820002002200420062008201020122014201620182020202220242026202820302032203420362038204020422044204620482050205220542056205820602062206420662068207020722074207620782080208220842086208820902092209420962098210021022104210621082110211221142116211821202122212421262128213021322134213621382140214221442146214821502152215421562158216021622164216621682170217221742176217821802182218421862188219021922194219621982200220222042206220822102212221422162218222022222224222622282230223222342236223822402242224422462248225022522254225622582260226222642266226822702272227422762278228022822284228622882290229222942296229823002302230423062308231023122314231623182320232223242326232823302332233423362338234023422344234623482350235223542356235823602362236423662368237023722374237623782380238223842386238823902392239423962398240024022404240624082410241224142416241824202422242424262428243024322434243624382440244224442446244824502452245424562458246024622464246624682470247224742476247824802482248424862488249024922494249624982500250225042506250825102512251425162518252025222524252625282530253225342536253825402542254425462548255025522554255625582560256225642566256825702572257425762578258025822584258625882590259225942596259826002602260426062608261026122614261626182620262226242626262826302632263426362638264026422644264626482650265226542656265826602662266426662668267026722674267626782680268226842686268826902692269426962698270027022704270627082710271227142716271827202722272427262728273027322734273627382740274227442746274827502752275427562758276027622764276627682770277227742776277827802782278427862788279027922794279627982800280228042806280828102812281428162818282028222824282628282830283228342836283828402842284428462848285028522854285628582860286228642866286828702872287428762878288028822884288628882890289228942896289829002902290429062908291029122914291629182920292229242926292829302932293429362938294029422944294629482950295229542956295829602962296429662968297029722974297629782980298229842986298829902992299429962998300030023004300630083010301230143016301830203022302430263028303030323034303630383040304230443046304830503052305430563058306030623064306630683070307230743076307830803082308430863088309030923094309630983100310231043106310831103112311431163118312031223124312631283130313231343136313831403142314431463148315031523154315631583160316231643166316831703172317431763178318031823184318631883190319231943196319832003202320432063208321032123214321632183220322232243226322832303232323432363238324032423244324632483250325232543256325832603262326432663268327032723274327632783280328232843286328832903292329432963298330033023304330633083310331233143316331833203322332433263328333033323334333633383340334233443346334833503352335433563358336033623364336633683370337233743376337833803382338433863388339033923394339633983400340234043406340834103412341434163418342034223424342634283430343234343436343834403442344434463448345034523454345634583460346234643466346834703472347434763478348034823484348634883490349234943496349835003502350435063508351035123514351635183520352235243526352835303532353435363538354035423544354635483550355235543556355835603562356435663568357035723574357635783580358235843586358835903592359435963598360036023604360636083610361236143616361836203622362436263628363036323634363636383640364236443646364836503652365436563658366036623664366636683670367236743676367836803682368436863688369036923694369636983700370237043706370837103712371437163718372037223724372637283730373237343736373837403742374437463748375037523754375637583760376237643766376837703772377437763778378037823784378637883790379237943796379838003802380438063808381038123814381638183820382238243826382838303832383438363838384038423844384638483850385238543856385838603862386438663868387038723874387638783880388238843886388838903892389438963898390039023904390639083910391239143916391839203922392439263928393039323934393639383940394239443946394839503952395439563958396039623964396639683970397239743976397839803982398439863988399039923994399639984000", "24681012141618202224262830323436384042444648505254565860626466687072747678808284868890929496981001021041061081101121141161181201221241261281301321341361381401421441461481501521541561581601621641661681701721741761781801821841861881901921941961982002022042062082102122142162182202222242262282302322342362382402422442462482502522542562582602622642662682702722742762782802822842862882902922942962983003023043063083103123143163183203223243263283303323343363383403423443463483503523543563583603623643663683703723743763783803823843863883903923943963984004024044064084104124144164184204224244264284304324344364384404424444464484504524544564584604624644664684704724744764784804824844864884904924944964985005025045065085105125145165185205225245265285305325345365385405425445465485505525545565585605625645665685705725745765785805825845865885905925945965986006026046066086106126146166186206226246266286306326346366386406426446466486506526546566586606626646666686706726746766786806826846866886906926946966987007027047067087107127147167187207227247267287307327347367387407427447467487507527547567587607627647667687707727747767787807827847867887907927947967988008028048068088108128148168188208228248268288308328348368388408428448468488508528548568588608628648668688708728748768788808828848868888908928948968989009029049069089109129149169189209229249269289309329349369389409429449469489509529549569589609629649669689709729749769789809829849869889909929949969981000100210041006100810101012101410161018102010221024102610281030103210341036103810401042104410461048105010521054105610581060106210641066106810701072107410761078108010821084108610881090109210941096109811001102110411061108111011121114111611181120112211241126112811301132113411361138114011421144114611481150115211541156115811601162116411661168117011721174117611781180118211841186118811901192119411961198120012021204120612081210121212141216121812201222122412261228123012321234123612381240124212441246124812501252125412561258126012621264126612681270127212741276127812801282128412861288129012921294129612981300130213041306130813101312131413161318132013221324132613281330133213341336133813401342134413461348135013521354135613581360136213641366136813701372137413761378138013821384138613881390139213941396139814001402140414061408141014121414141614181420142214241426142814301432143414361438144014421444144614481450145214541456145814601462146414661468147014721474147614781480148214841486148814901492149414961498150015021504150615081510151215141516151815201522152415261528153015321534153615381540154215441546154815501552155415561558156015621564156615681570157215741576157815801582158415861588159015921594159615981600160216041606160816101612161416161618162016221624162616281630163216341636163816401642164416461648165016521654165616581660166216641666166816701672167416761678168016821684168616881690169216941696169817001702170417061708171017121714171617181720172217241726172817301732173417361738174017421744174617481750175217541756175817601762176417661768177017721774177617781780178217841786178817901792179417961798180018021804180618081810181218141816181818201822182418261828183018321834183618381840184218441846184818501852185418561858186018621864186618681870187218741876187818801882188418861888189018921894189618981900190219041906190819101912191419161918192019221924192619281930193219341936193819401942194419461948195019521954195619581960196219641966196819701972197419761978198019821984198619881990199219941996199820002002200420062008201020122014201620182020202220242026202820302032203420362038204020422044204620482050205220542056205820602062206420662068207020722074207620782080208220842086208820902092209420962098210021022104210621082110211221142116211821202122212421262128213021322134213621382140214221442146214821502152215421562158216021622164216621682170217221742176217821802182218421862188219021922194219621982200220222042206220822102212221422162218222022222224222622282230223222342236223822402242224422462248225022522254225622582260226222642266226822702272227422762278228022822284228622882290229222942296229823002302230423062308231023122314231623182320232223242326232823302332233423362338234023422344234623482350235223542356235823602362236423662368237023722374237623782380238223842386238823902392239423962398240024022404240624082410241224142416241824202422242424262428243024322434243624382440244224442446244824502452245424562458246024622464246624682470247224742476247824802482248424862488249024922494249624982500250225042506250825102512251425162518252025222524252625282530253225342536253825402542254425462548255025522554255625582560256225642566256825702572257425762578258025822584258625882590259225942596259826002602260426062608261026122614261626182620262226242626262826302632263426362638264026422644264626482650265226542656265826602662266426662668267026722674267626782680268226842686268826902692269426962698270027022704270627082710271227142716271827202722272427262728273027322734273627382740274227442746274827502752275427562758276027622764276627682770277227742776277827802782278427862788279027922794279627982800280228042806280828102812281428162818282028222824282628282830283228342836283828402842284428462848285028522854285628582860286228642866286828702872287428762878288028822884288628882890289228942896289829002902290429062908291029122914291629182920292229242926292829302932293429362938294029422944294629482950295229542956295829602962296429662968297029722974297629782980298229842986298829902992299429962998300030023004300630083010301230143016301830203022302430263028303030323034303630383040304230443046304830503052305430563058306030623064306630683070307230743076307830803082308430863088309030923094309630983100310231043106310831103112311431163118312031223124312631283130313231343136313831403142314431463148315031523154315631583160316231643166316831703172317431763178318031823184318631883190319231943196319832003202320432063208321032123214321632183220322232243226322832303232323432363238324032423244324632483250325232543256325832603262326432663268327032723274327632783280328232843286328832903292329432963298330033023304330633083310331233143316331833203322332433263328333033323334333633383340334233443346334833503352335433563358336033623364336633683370337233743376337833803382338433863388339033923394339633983400340234043406340834103412341434163418342034223424342634283430343234343436343834403442344434463448345034523454345634583460346234643466346834703472347434763478348034823484348634883490349234943496349835003502350435063508351035123514351635183520352235243526352835303532353435363538354035423544354635483550355235543556355835603562356435663568357035723574357635783580358235843586358835903592359435963598360036023604360636083610361236143616361836203622362436263628363036323634363636383640364236443646364836503652365436563658366036623664366636683670367236743676367836803682368436863688369036923694369636983700370237043706370837103712371437163718372037223724372637283730373237343736373837403742374437463748375037523754375637583760376237643766376837703772377437763778378037823784378637883790379237943796379838003802380438063808381038123814381638183820382238243826382838303832383438363838384038423844384638483850385238543856385838603862386438663868387038723874387638783880388238843886388838903892389438963898390039023904390639083910391239143916391839203922392439263928393039323934393639383940394239443946394839503952395439563958396039623964396639683970397239743976397839803982398439863988399039923994399639984000"],+ + "oldkeys": { + + "keynames": ["id"], + + "keytypes": ["int4"], + + "keyvalues": [1] + + } + + } + + ] + + } +(4 rows) + +SELECT 'stop' FROM pg_drop_replication_slot('regression_slot'); + ?column? +---------- + stop +(1 row) + diff --git a/sql/missing_toast.sql b/sql/missing_toast.sql new file mode 100644 index 0000000..4a17682 --- /dev/null +++ b/sql/missing_toast.sql @@ -0,0 +1,30 @@ +\set VERBOSITY terse + +-- predictability +SET synchronous_commit = on; + +DROP TABLE IF EXISTS missing_toast; + +CREATE TABLE missing_toast ( +id serial primary key, +an_integer integer, +toasted_col1 text, +toasted_col2 text +); + +SELECT 'init' FROM pg_create_logical_replication_slot('regression_slot', 'wal2json'); + +-- uncompressed external toast data +INSERT INTO missing_toast (toasted_col1, toasted_col2) SELECT string_agg(g.i::text, ''), string_agg((g.i*2)::text, '') FROM generate_series(1, 2000) g(i); + +-- update of existing column +UPDATE missing_toast SET toasted_col1 = (SELECT string_agg((g.i*2)::text, '') FROM generate_series(1, 2000) g(i)) where id = 1; + +UPDATE missing_toast set an_integer = 1 where id = 1; + +UPDATE missing_toast SET toasted_col1 = (SELECT string_agg((g.i*2)::text, '') FROM generate_series(1, 2000) g(i)), +toasted_col2 = (SELECT string_agg((g.i*2)::text, '') FROM generate_series(1, 2000) g(i)) +where id = 1; + +SELECT data FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'pretty-print', '1', 'include-typmod', '0', 'include-missing-toast', '1'); +SELECT 'stop' FROM pg_drop_replication_slot('regression_slot');