CRASH at 0x..96c1 on appveyor in various tests

I can't reproduce these crashes locally on a win8.1 or other machine.

Appveyor 64-bit has crashed in registers.exe, malloc.exe, and pcache.exe. The crashes always have the address 0x..96c1. Here is an example:

[00:06:17]   8/114 Test   #7: registers .........................***Failed    1.83 sec
[00:06:17] STDOUT: 
[00:06:17] STDERR: <Starting application C:\projects\drmemory\build\build_drmemory-dbg-64\tests\registers.exe (864)>
[00:06:17] <Initial options = -no_dynamic_options -logdir 'C:\projects\drmemory\build\build_drmemory-dbg-64\logs\dynamorio' -client_lib 'C:\projects\drmemory\build\build_drmemory-dbg-64\bin64\debug\drmemorylib.dll;0;`-callstack_style` `0x27` -no_results_to_stderr -logdir `C:\projects\drmemory\build\build_drmemory-dbg-64\logs` -symcache_dir `C:\projects\drmemory\build\build_drmemory-dbg-64\logs\symcache` -lib_blacklist `C:\windows*.d??,C:\Program Files\Common Files\Microsoft Shared*.d??,C:\Program Files (x86)\Common Files\Microsoft Shared*.d??` -resfile 864 ' -code_api -probe_api -msgbox_mask 0 -dumpcore_mask 0x87fd -stack_size 56K -disable_traces -no_enable_traces -max_elide_jmp 0 -max_elide_call 0 -no_shared_traces -bb_ibl_targets -bb_single_restore_prefix -no_shared_trace_ibl_routine -no_enable_reset -no_reset_at_switch_to_os_at_vmm_limit -reset_at_vmm_percent_free_limit 0 -no_reset_at_vmm_full -reset_at_commit_free_limit 0K -reset_every_nth_pending 0 -no_vm_base_near_app -no_inline_ignored_syscalls -native_exec_default_list '' -no_native_exec_managed_code -no_indcall2direct -pad_jmps_mark_no_trace >
[00:06:17] ~~Dr.M~~ Dr. Memory version 1.11.17967
[00:06:17] ~~Dr.M~~ Running "C:/projects/drmemory/build/build_drmemory-dbg-64/tests/registers.exe"
[00:06:17] <Core dump file for application C:\projects\drmemory\build\build_drmemory-dbg-64\tests\registers.exe (864) created at C:\projects\drmemory\build\build_drmemory-dbg-64\logs\dynamorio\registers.exe.864.00000000.ldmp>
[00:06:17] <Application C:\projects\drmemory\build\build_drmemory-dbg-64\tests\registers.exe (864).  Dr. Memory internal crash at PC 0x00000000000996c1.  Please report this at http://drmemory.org/issues.  Program aborted.
[00:06:17] 0xc0000005 0x00000000 0x00000000000996c1 0x00000000000996c1 0x0000000000000000 0x000004007c250000
[00:06:17] Base: 0x0000000015000000
[00:06:17] Registers: eax=0x0000000000000000 ebx=0x0000000000000000 ecx=0x000004007c250000 edx=0x0000000000000000
[00:06:17]      esi=0x00000001f04ce590 edi=0x00000001f04cb3a0 esp=0x00000001f04cb300 ebp=0x0000000000000000
[00:06:17]      r8 =0x0000000000000f84 r9 =0x00000001f04cb3f8 r10=0x0000000000000000 r11=0x0000000000000246
[00:06:17]      r12=0x0000000000000000 r13=0x0000000000000000 r14=0x0000000000000000 r15=0x0000000000000000
[00:06:17]      eflags=0xcccccccc00010246
[00:06:17] 1.11.17967-1-(Mar 12 2019 05:09:04) WinVer=63;Rel=;Build=9600;Edition=ServerDatacenter
[00:06:23] -no_dynamic_options -logdir 'C:\projects\drmemory\build\build_drmemory-dbg-64\logs\dynamorio' -client_lib 'C:\projects\drmemory\build\build_drmemory-dbg-64\bin64\debug\drmemorylib.dll;0;`-callstack_style` `0x27` -no_results_to_stderr -logdir `C:\projects\drmemory\build\build_drmemory-dbg-64\logs` -symcache_dir `C:\projec
[00:06:23] C:\projects\drmemory\build\build_drmemory-dbg-64\bin64\debug\drmemorylib.dll=0x0000000000010000
[00:06:23] C:\projects\drmemory\build\build_drmemory-dbg-64\bin64\debug/dbghelp.dll=0x0000000000870000
[00:06:23] C:\windows/system32/msvcrt.dll=0x00007ffc840b0000
[00:06:23] C:\windows/system32/kernel32.dll=0x00000001f06d0000
[00:06:23] C:\windows/system32/KERNELBASE.dll=0x00000001f0810000>

I did special runs with verbosity and logs to stderr in PR #2174 and reproduced with -verbose 2 (unfortunately not with higher) in registers.exe. Here is the output from the prior error report to the crash:

#+BEGIN_EXAMPLE
[00:06:18] Error #13: UNINITIALIZED READ: reading 0x000000297263fe10-0x000000297263fe18 8 byte(s)
[00:06:18] # 0 cmpxchg8b_test_asm [C:\projects\drmemory\build\build_drmemory-dbg-64\tests\CMakeFiles\registersB.dir\registers.c_asm.asm.obj.s:1214] (0x0000
7ff6824518a6 <registersB.exe+0x18a6>) modid:3
[00:06:18] # 1 cmpxchg8b_test [c:\projects\drmemory\tests\registers.c:153] (0x00007ff682451346 <registersB.exe+0x1346>) modid:3
[00:06:18] # 2 main           [c:\projects\drmemory\tests\registers.c:324] (0x00007ff682451815 <registersB.exe+0x1815>) modid:3
[00:06:18] Note: @0:00:03.828 in thread 748
[00:06:18] Note: instruction: cmpxchg8b (%rcx) %eax %edx %ecx %ebx -> (%rcx) %eax %edx
[00:06:18]      error end
[00:06:18] in event_basic_block(tag=0x00007ff682451346)
[00:06:18] set range 0x000000297218fb50-0x000000297218fb58 => 0x0
[00:06:18] replace_free 0x000000297263fe10
[00:06:18] set range 0x000000297263fe10-0x000000297263fe18 => 0x1
[00:06:18]      replace_free_common 0x000000297263fe10 == request=8, alloc=16, arena=0x0000002972630000
[00:06:18] set range 0x000000297218fb50-0x000000297218fb58 => 0x1
[00:06:18] in event_basic_block(tag=0x00007ff682451350)
[00:06:18] set range 0x000000297218fb50-0x000000297218fb58 => 0x0
[00:06:18] replace_free 0x000000297263fe40
[00:06:18] set range 0x000000297263fe40-0x000000297263fe48 => 0x1
[00:06:18]      replace_free_common 0x000000297263fe40 == request=8, alloc=16, arena=0x0000002972630000
[00:06:18] set range 0x000000297218fb50-0x000000297218fb58 => 0x1
[00:06:18] in event_basic_block(tag=0x00007ff68245135a)
[00:06:18] set range 0x000000297218fb60-0x000000297218fb98 => 0x1
[00:06:18] in event_basic_block(tag=0x00007ff682451815)
[00:06:18] in event_basic_block(tag=0x00007ff682451370)
[00:06:18] set range 0x000000297218fb60-0x000000297218fb98 => 0x3
[00:06:18] set range 0x000000297218fb50-0x000000297218fb58 => 0x0
[00:06:18] replace_malloc (nomatch) 8
[00:06:18]      carving out new chunk @0x000000297263fe50 => head=0x000000297263fe50, res=0x000000297263fe70
[00:06:18]      replace_alloc_common arena=0x0000002972630000 flags=0x1004 request=8, align=16 alloc=16 => 0x000000297263fe70
[00:06:18] set range 0x000000297263fe70-0x000000297263fe78 => 0x3
[00:06:18]      replace_malloc 8 => 0x000000297263fe70
[00:06:18] set range 0x000000297218fb50-0x000000297218fb58 => 0x1
[00:06:18] in event_basic_block(tag=0x00007ff68245137e)
[00:06:18] in event_basic_block(tag=0x00007ff6824518af)
[00:06:18] in event_basic_block(tag=0x00007ff682451394)
[00:06:18] set range 0x000000297218fb50-0x000000297218fb58 => 0x0
[00:06:18] replace_free 0x000000297263fe70
[00:06:18] set range 0x000000297263fe70-0x000000297263fe78 => 0x1
[00:06:18]      replace_free_common 0x000000297263fe70 == request=8, alloc=16, arena=0x0000002972630000
[00:06:18] set range 0x000000297218fb50-0x000000297218fb58 => 0x1
[00:06:18] in event_basic_block(tag=0x00007ff68245139e)
[00:06:18] set range 0x000000297218fb60-0x000000297218fb98 => 0x1
[00:06:18] in event_basic_block(tag=0x00007ff68245181a)
[00:06:18] in event_basic_block(tag=0x00007ff6824513b0)
[00:06:18] set range 0x000000297218fb60-0x000000297218fb98 => 0x3
[00:06:18] in event_basic_block(tag=0x00007ff682451a7c)
[00:06:18] in event_basic_block(tag=0x00007ff6824513c7)
[00:06:18] set range 0x000000297218fb60-0x000000297218fb98 => 0x1
[00:06:18] in event_basic_block(tag=0x00007ff68245181f)
[00:06:18] in event_basic_block(tag=0x00007ff682451470)
[00:06:18] set range 0x000000297218fae0-0x000000297218fb98 => 0x3
[00:06:18] set range 0x000000297218fad0-0x000000297218fad8 => 0x0
[00:06:18] replace_malloc (nomatch) 128
[00:06:18]      carving out new chunk @0x000000297263fe80 => head=0x000000297263fe80, res=0x000000297263fea0
[00:06:18]      replace_alloc_common arena=0x0000002972630000 flags=0x1004 request=128, align=16 alloc=128 => 0x000000297263fea0
[00:06:18] set range 0x000000297263fea0-0x000000297263ff20 => 0x3
[00:06:18]      replace_malloc 128 => 0x000000297263fea0
[00:06:18] set range 0x000000297218fad0-0x000000297218fad8 => 0x1
[00:06:18] in event_basic_block(tag=0x00007ff682451481)
[00:06:18] set range 0x000000297218faa0-0x000000297218fac8 => 0x3
[00:06:18] set range 0x000000297218fa70-0x000000297218fa90 => 0x3
<...>
[00:06:18] set range 0x000000297218fa70-0x000000297218fa90 => 0x1
[00:06:18] set range 0x000000297218faa0-0x000000297218fac8 => 0x1
[00:06:18] in event_basic_block(tag=0x00007ff6824514a0)
[00:06:18] in event_basic_block(tag=0x00007ff682451960)
[00:06:18] <Core dump file for application C:\projects\drmemory\build\build_drmemory-dbg-64\tests\registersB.exe (2484) created at C:\projects\drmemory\build\build_drmemory-dbg-64\logs\dynamorio\registersB.exe.2484.00000000.ldmp>
[00:06:18] <Application C:\projects\drmemory\build\build_drmemory-dbg-64\tests\registersB.exe (2484).  Dr. Memory internal crash at PC 0x00000000000996b1.  Please report this at http://drmemory.org/issues.  Program aborted.
[00:06:18] 0xc0000005 0x00000000 0x00000000000996b1 0x00000000000996b1 0x0000000000000000 0x0000040a5c990000
[00:06:18] Base: 0x0000000015000000
[00:06:18] Registers: eax=0x0000000000000000 ebx=0x0000000000000000 ecx=0x0000040a5c990000 edx=0x0000000000000000
[00:06:18]      esi=0x00000029721de590 edi=0x00000029721db3a0 esp=0x00000029721db300 ebp=0x0000000000000000
[00:06:18]      r8 =0x0000000000014b93 r9 =0x00000029721db3f8 r10=0x0000000000000000 r11=0x0000000000000246
[00:06:18]      r12=0x0000000000000000 r13=0x0000000000000000 r14=0x0000000000000000 r15=0x0000000000000000
[00:06:18]      eflags=0xcccccccc00010246
[00:06:18] 1.11.17972-1-(Mar 17 2019 01:26:36) WinVer=63;Rel=;Build=9600;Edition=ServerDatacenter
[00:06:18] -no_dynamic_options -logdir 'C:\projects\drmemory\build\build_drmemory-dbg-64\logs\dynamorio' -client_lib 'C:\projects\drmemory\build\build_drmemory-dbg-64\bin64\debug\drmemorylib.dll;0;`-callstack_style` `0x27` -no_results_to_stderr `-verbose` `2` -logdir `C:\projects\drmemory\build\build_drmemory-dbg-64\logs` -symcache
[00:06:18] C:\projects\drmemory\build\build_drmemory-dbg-64\bin64\debug\drmemorylib.dll=0x0000000000010000
[00:06:18] C:\projects\drmemory\build\build_drmemory-dbg-64\bin64\debug/dbghelp.dll=0x0000000000870000
[00:06:18] C:\windows/system32/msvcrt.dll=0x00007fff7dbe0000
[00:06:18] C:\windows/system32/kernel32.dll=0x0000002972360000
[00:06:18] C:\windows/system32/KERNELBASE.dll=0x0000002972510000>
[00:06:18] ~~Dr.M~~ WARNING: application exited with abnormal code 0xffffffff

0x00007ff682451346 <registersB.exe+0x1346>) modid:3 => final bb is: 0x00007ff682451960 == registersB.exe+0x1960

Looks like my local build lines up:

0:000> Uf registers!cmpxchg8b_test
registers!cmpxchg8b_test [d:\derek\drmemory\git\src\tests\registers.c @ 150]:
  150 00007ff7`619f1310 4883ec38        sub     rsp,38h
  151 00007ff7`619f1314 b908000000      mov     ecx,8
  151 00007ff7`619f1319 e8820b0000      call    registers!malloc (00007ff7`619f1ea0)
  151 00007ff7`619f131e 4889442420      mov     qword ptr [rsp+20h],rax
  152 00007ff7`619f1323 ba01000000      mov     edx,1
  152 00007ff7`619f1328 b908000000      mov     ecx,8
  152 00007ff7`619f132d e8ea0a0000      call    registers!calloc (00007ff7`619f1e1c)
  152 00007ff7`619f1332 4889442428      mov     qword ptr [rsp+28h],rax
  153 00007ff7`619f1337 488b542420      mov     rdx,qword ptr [rsp+20h]
  153 00007ff7`619f133c 488b4c2428      mov     rcx,qword ptr [rsp+28h]
  153 00007ff7`619f1341 e850050000      call    registers!cmpxchg8b_test_asm (00007ff7`619f1896)
  154 00007ff7`619f1346 488b4c2420      mov     rcx,qword ptr [rsp+20h]

So:

0:000> U registers+14a0
registers!addronly_test+0x30 [d:\derek\drmemory\git\src\tests\registers.c @ 203]:
00007ff7`619f14a0 488b4c2420      mov     rcx,qword ptr [rsp+20h]
00007ff7`619f14a5 e8b6040000      call    registers!addronly_test_asm (00007ff7`619f1960)

registers!addronly_test_asm [D:\derek\drmemory\git\build_DRtoo_x64_dbg\tests\CMakeFiles\registers.dir\registers.c_asm.asm.obj.s @ 1371]:
 1371 00007ff7`619f1960 488bc1          mov     rax,rcx
 1375 00007ff7`619f1963 53              push    rbx
 1380 00007ff7`619f1964 fe8080000000    inc     byte ptr [rax+80h]
 1381 00007ff7`619f196a 48ffc0          inc     rax
 1382 00007ff7`619f196d 48ffc0          inc     rax
 1383 00007ff7`619f1970 48ffc0          inc     rax
 1384 00007ff7`619f1973 48ffc0          inc     rax
 1385 00007ff7`619f1976 48ffc1          inc     rcx
 1386 00007ff7`619f1979 48ffc1          inc     rcx
 1387 00007ff7`619f197c 48ffc1          inc     rcx
 1388 00007ff7`619f197f 48ffc1          inc     rcx
 1389 00007ff7`619f1982 eb00            jmp     registers!addronly_test_asm+0x24 (00007ff7`619f1984)  Branch

registers!addronly_test_asm+0x24 [D:\derek\drmemory\git\build_DRtoo_x64_dbg\tests\CMakeFiles\registers.dir\registers.c_asm.asm.obj.s @ 1392]:
 1392 00007ff7`619f1984 fe8080000000    inc     byte ptr [rax+80h]
 1393 00007ff7`619f198a 48ffc3          inc     rbx

Hmm, I don't see how that bb would cause a crash (and it doesn't locally). Stumped.