Hi,
I am extending a working VDDK C++ application (similar to the VDDK sample program) with SWIG using Python. I.e., the C++ code and a SWIG .i file were compiled and linked as a .so lib and a Python script simply calls it to perform the intended VDDK task. The program SIGSEGVed in /usr/lib/vmware-vix-disklib/lib64/libcrypto.so.0.9.8 while calling VixDiskLib_PrepareForAccess(). The error message is not obvious to me for what is missing. Strace indicated that it core-dumped while accessing the "/etc/ssl/certs/ca-certificates.crt" file.
====== Program received signal SIGSEGV with the following gdb backtrace.
__memcmp_sse4_1 () at ../sysdeps/x86_64/multiarch/memcmp-sse4.S:1523
1523 ../sysdeps/x86_64/multiarch/memcmp-sse4.S: No such file or directory.
(gdb) bt
#0 __memcmp_sse4_1 () at ../sysdeps/x86_64/multiarch/memcmp-sse4.S:1523
#1 0x00007ffff428ae32 in OBJ_bsearch_ex ()
from /usr/lib/vmware-vix-disklib/lib64/libcrypto.so.0.9.8
...
#21 0x00007ffff5961b80 in VixDiskLib_PrepareForAccess ()
from /usr/lib/vmware-vix-disklib/lib64/libvixDiskLib.so.5
#22 0x00007ffff5d60c78 in DiskLib_Init(unsigned long, bool, VixDiskLibConnectParams&) () from src/_snapcacheswig.so
#23 0x00007ffff5d60ef2 in ProcessCommand(int) ()
from src/_snapcacheswig.so <=== this is the .so file called by Python
...
The other strange symptoms is that it seems to be accessing "/etc/ssl/certs/ca-certificates.crt" when the program core-dumped. The working c++ code while linked with the same libraries, it does not seem to access the cert file. Or no issue accessing it.
write(1, "Log: VixDiskLibVim: Login comple"..., 36Log: VixDiskLibVim: Login completed
) = 36
write(1, "Use glog\n", 9Use glog
) = 9
poll([{fd=6, events=POLLOUT}, {fd=3, events=POLLIN}], 2, 3) = 0 (Timeout)
poll([{fd=6, events=POLLOUT|POLLWRNORM}], 1, 0) = 0 (Timeout)
poll([{fd=6, events=POLLOUT}, {fd=3, events=POLLIN}], 2, 1000) = 1 ([{fd=6, revents=POLLOUT}])
poll([{fd=6, events=POLLOUT|POLLWRNORM}], 1, 0) = 1 ([{fd=6, revents=POLLOUT|POLLWRNORM}])
getsockopt(6, SOL_SOCKET, SO_ERROR, [0], [4]) = 0
getpeername(6, {sa_family=AF_INET, sin_port=htons(443), sin_addr=inet_addr("192.168.1.2")}, [16]) = 0
getsockname(6, {sa_family=AF_INET, sin_port=htons(34940), sin_addr=inet_addr("192.168.168.171")}, [16]) = 0
stat("/dev/urandom", {st_mode=S_IFCHR|0666, st_rdev=makedev(1, 9), ...}) = 0
open("/dev/urandom", O_RDONLY) = 7
read(7, "\346\303\226\345\253\364\360\341r`\304dZ\"\210\301\16q\211\374\364\331\377\6\250b\3029\31\254/\27"..., 1024) = 1024
close(7) = 0
open("/etc/ssl/certs/ca-certificates.crt", O_RDONLY) = 7
fstat(7, {st_mode=S_IFREG|0644, st_size=255550, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fbfe44b3000
read(7, "-----BEGIN CERTIFICATE-----\nMIID"..., 4096) = 4096
...
read(7, "\n-----END CERTIFICATE-----\n-----"..., 4096) = 4096
...
brk(0xeb1000) = 0xeb1000
read(7, "\n1UkC9gLl9B/rfNmWVan/7Ir5mUf/NVo"..., 4096) = 4096
...
read(7, "fwxArNK4aUyeNgsURSsloptJGXg9i3ph"..., 4096) = 1598
read(7, "", 4096) = 0
gettid() = 12147
gettid() = 12147
gettid() = 12147
close(7) = 0
munmap(0x7fbfe44b3000, 4096) = 0
--- SIGSEGV (Segmentation fault) @ 0 (0) ---
+++ killed by SIGSEGV (core dumped) +++
Segmentation fault (core dumped)
Appreciate any pointer to debug this issue.
-JL
=======================================================
Additional information as follows:
Dev environment:
- Ubuntu 12.04 with g++
- VMware VixDiskLib (5.5) Release build-1601882
- VCenter server appliance (VMware-vCenter-Server-Appliance-5.5.0.5201-1476389)
- ESXi
=================== The build/Make command used are:
$ make allso
protoc --cpp_out=../src-gen --java_out=../src-gen --python_out=../src-gen multivmsnapshot.proto
g++ -Wall -m64 -fPIC -c protobuf_util.cc ../src-gen/multivmsnapshot.pb.cc -I../src-gen `pkg-config --cflags protobuf`
rm -rf *.java snapcacheswig.py snapcacheswig.pyc snapcacheswig_wrap.cxx
swig -java -c++ snapcacheswig.i
swig -python -c++ snapcacheswig.i
g++ -Wall -m64 -fPIC -c snapcachevddk.cc ../src-gen/multivmsnapshot.pb.cc -I/usr/local/include/glog -I../src-gen `pkg-config --cflags protobuf` `pkg-config --cflags vix-disklib`
snapcachevddk.cc:517:1: warning: ‘int PrintUsage()’ defined but not used [-Wunused-function]
g++ -Wall -m64 -fPIC -c snapcacheswig_wrap.cxx -I. -I../src-gen `pkg-config --cflags vix-disklib` -I/usr/include/python2.7
g++ -Wall -m64 --shared -o _snapcacheswig.so snapcachevddk.o protobuf_util.o multivmsnapshot.pb.o snapcacheswig_wrap.o -Wl,-rpath=/usr/lib/vmware-vix-disklib/lib64,-rpath=/usr/local/lib,-rpath=/usr/lib/x86_64-linux-gnu `pkg-config --libs vix-disklib` `pkg-config --libs protobuf` -L/usr/lib/x86_64-linux-gnu -L/usr/local/lib -lglog
==================== The library rpath appears to be correct.
$ readelf -d _snapcacheswig.so
Dynamic section at offset 0x83ae8 contains 29 entries:
Tag Type Name/Value
0x0000000000000001 (NEEDED) Shared library: [libvixDiskLib.so.5]
0x0000000000000001 (NEEDED) Shared library: [libprotobuf.so.8]
0x0000000000000001 (NEEDED) Shared library: [libpthread.so.0]
0x0000000000000001 (NEEDED) Shared library: [libglog.so.0]
0x0000000000000001 (NEEDED) Shared library: [libstdc++.so.6]
0x0000000000000001 (NEEDED) Shared library: [libgcc_s.so.1]
0x0000000000000001 (NEEDED) Shared library: [libc.so.6]
0x000000000000000f (RPATH) Library rpath: [/usr/lib/vmware-vix-disklib/lib64:/usr/local/lib:/usr/lib/x86_64-linux-gnu]
0x000000000000000c (INIT) 0x33be0
0x000000000000000d (FINI) 0x6c608
0x0000000000000019 (INIT_ARRAY) 0x282c88
0x000000000000001b (INIT_ARRAYSZ) 32 (bytes)
0x000000006ffffef5 (GNU_HASH) 0x1f0
0x0000000000000005 (STRTAB) 0xcac0
0x0000000000000006 (SYMTAB) 0x2c68
0x000000000000000a (STRSZ) 110928 (bytes)
0x000000000000000b (SYMENT) 24 (bytes)
0x0000000000000003 (PLTGOT) 0x283fe8
0x0000000000000002 (PLTRELSZ) 28128 (bytes)
0x0000000000000014 (PLTREL) RELA
0x0000000000000017 (JMPREL) 0x2ce00
0x0000000000000007 (RELA) 0x28a38
0x0000000000000008 (RELASZ) 17352 (bytes)
0x0000000000000009 (RELAENT) 24 (bytes)
0x000000006ffffffe (VERNEED) 0x28948
0x000000006fffffff (VERNEEDNUM) 5
0x000000006ffffff0 (VERSYM) 0x27c10
0x000000006ffffff9 (RELACOUNT) 241
0x0000000000000000 (NULL) 0x0
====================== gdb BT as follows
...
Log: VixDiskLib: linuxVerifySSLCertificates is 1
Use glog
Log: VixDiskLib: config options: libdir '<null>', tmpDir '<null>'.
Use glog
Log: VixDiskLib: Initialize Disklib.
Use glog
Log: OBJLIB-LIB: Objlib initialized.
Use glog
Log: VixDiskLib: Available transport modes from DiskLib: file:nbdssl:nbd.
Use glog
Log: VixDiskLib: Advanced transport module not loaded.
Use glog
Log: VixDiskLib: Using transport modes from DiskLib: file:nbdssl:nbd.
Use glog
Log: VMware VixDiskLib (5.5) Release build-1601882
Use glog
Log: VixDiskLib: Initializing SSL.
Use glog
Log: VixDiskLib: Initializing NFC.
Use glog
Warning: program compiled against libxml 208 using older 207
Log: VixDiskLibVim: VixDiskLibVim_Init: Initialization is completed.
Use glog
I0417 15:34:18.301712 11452 snapcachevddk.cc:612] here 1
Log: VixDiskLib: VixDiskLib_PrepareForAccess: Prepare to access disk.
Use glog
Log: VixDiskLib: VixDiskLib_Connect: Establish connection.
Use glog
Log: VixDiskLib: VixDiskLib_Connect: Use remote connection.
Use glog
Log: VixDiskLib: Allocate connection.
Use glog
Log: VixDiskLib: Prepare remote connect.
Use glog
Log: VixDiskLib: Resolve host 192.168.x.xx (masked the IP ...)
Use glog
Log: VixDiskLib: Retrieved 192.168.x.xx
Use glog
Log: VixDiskLib: Load thumbprint.
Use glog
Log: VixDiskLib: Open sslIdDb.
Use glog
Log: VixDiskLib: Add thumbprint.
Use glog
Log: VixDiskLib: Added thumbprint CB:F3:1A:7B:93:02:20:....:66:70:B4:F5 to sslIdDb for 192.168.x.xx (the thumbprint is masked)
Use glog
Log: VixDiskLib: Close sslIdDb.
Use glog
Log: VixDiskLib: Release sslIdDb.
Use glog
Log: VixDiskLib: VixDiskLib_PrepareForAccess: Temporarily disable Storage VMotion.
Use glog
Log: VixDiskLibVim: VixDiskLibVim_AllowVMotion: Disable VMotion.
Use glog
Log: VixDiskLibVim: Resolve host.
Use glog
Log: VixDiskLibVim: Resolving IP address for host 192.168.x.xx
Use glog
Log: VixDiskLibVim: Resolved to 192.168.x.xx
Use glog
Log: VixDiskLibVim: Clean up addrinfo.
Use glog
Log: VixDiskLibVim: Initialize callback data with username/passwd credential type.
Use glog
Log: VixDiskLibVim: Check if SSL verification is enabled.
Use glog
Log: VixDiskLibVim: SSL verification is enabled.
Use glog
Log: VixDiskLibVim: Initialize SSL.
Use glog
Log: VixDiskLibVim: Parse VMX.
Use glog
Log: VixDiskLibVim: Extract path from VMX.
Use glog
Log: VixDiskLibVim: Extract datastore from VMX path.
Use glog
Log: VixDiskLibVim: Skip leading character(s).
Use glog
Log: VixDiskLibVim: Unable to retrieve key from VMX.
Use glog
Log: VixDiskLibVim: Extract VM inventory path.
Use glog
Log: VixDiskLibVim: Find VM via inventory path.
Use glog
Log: VixDiskLibVim: Retrieved VM location nimbilab/vm/vddk2.
Use glog
Log: VixDiskLibVim: Connect to server.
Use glog
Log: VixDiskLibVim: Login via connection to VIM.
Use glog
Log: VixDiskLibVim: Login completed
Use glog
Program received signal SIGSEGV, Segmentation fault.
__memcmp_sse4_1 () at ../sysdeps/x86_64/multiarch/memcmp-sse4.S:1523
1523 ../sysdeps/x86_64/multiarch/memcmp-sse4.S: No such file or directory.
(gdb) bt
#0 __memcmp_sse4_1 () at ../sysdeps/x86_64/multiarch/memcmp-sse4.S:1523
#1 0x00007ffff428ae32 in OBJ_bsearch_ex ()
from /usr/lib/vmware-vix-disklib/lib64/libcrypto.so.0.9.8
#2 0x00007ffff42e0465 in sk_find ()
from /usr/lib/vmware-vix-disklib/lib64/libcrypto.so.0.9.8
#3 0x00007ffff43120f2 in X509_OBJECT_retrieve_match ()
from /usr/lib/vmware-vix-disklib/lib64/libcrypto.so.0.9.8
#4 0x00007ffff4312350 in X509_STORE_add_cert ()
from /usr/lib/vmware-vix-disklib/lib64/libcrypto.so.0.9.8
#5 0x00007ffff4313ad4 in X509_load_cert_crl_file ()
from /usr/lib/vmware-vix-disklib/lib64/libcrypto.so.0.9.8
#6 0x00007ffff4313bde in ?? ()
from /usr/lib/vmware-vix-disklib/lib64/libcrypto.so.0.9.8
#7 0x00007ffff2e287fa in SSL_SetVerifyCb ()
from /usr/lib/vmware-vix-disklib/lib64/libgvmomi.so.0
#8 0x00007ffff2a5c80f in ?? ()
from /usr/lib/vmware-vix-disklib/lib64/libgvmomi.so.0
#9 0x00007ffff20de096 in ?? () from /usr/lib/x86_64-linux-gnu/libcurl.so.4
#10 0x00007ffff20f20f8 in Curl_ssl_connect_nonblocking ()
from /usr/lib/x86_64-linux-gnu/libcurl.so.4
#11 0x00007ffff20c884e in ?? () from /usr/lib/x86_64-linux-gnu/libcurl.so.4
#12 0x00007ffff20d9bea in Curl_protocol_connect ()
from /usr/lib/x86_64-linux-gnu/libcurl.so.4
#13 0x00007ffff20ed183 in ?? () from /usr/lib/x86_64-linux-gnu/libcurl.so.4
#14 0x00007ffff20edccf in ?? () from /usr/lib/x86_64-linux-gnu/libcurl.so.4
#15 0x00007ffff20eddaf in curl_multi_socket_action ()
from /usr/lib/x86_64-linux-gnu/libcurl.so.4
#16 0x00007ffff2a5eb26 in ?? ()
from /usr/lib/vmware-vix-disklib/lib64/libgvmomi.so.0
#17 0x00007ffff3e9070b in ?? ()
from /usr/lib/vmware-vix-disklib/lib64/libvixDiskLibVim.so
#18 0x00007ffff3e9075f in ?? ()
from /usr/lib/vmware-vix-disklib/lib64/libvixDiskLibVim.so
#19 0x00007ffff3e91172 in ?? ()
from /usr/lib/vmware-vix-disklib/lib64/libvixDiskLibVim.so
#20 0x00007ffff3e8bff5 in VixDiskLibVim_AllowVMotion ()
from /usr/lib/vmware-vix-disklib/lib64/libvixDiskLibVim.so
#21 0x00007ffff5961b80 in VixDiskLib_PrepareForAccess ()
from /usr/lib/vmware-vix-disklib/lib64/libvixDiskLib.so.5
#22 0x00007ffff5d60c78 in DiskLib_Init(unsigned long, bool, VixDiskLibConnectParams&) () from
src/_snapcacheswig.so
#23 0x00007ffff5d60ef2 in ProcessCommand(int) ()
src/_snapcacheswig.so
---Type <return> to continue, or q <return> to quit---
#24 0x00007ffff5d93397 in _wrap_ProcessCommand ()
from src/_snapcacheswig.so
#25 0x000000000054bb14 in PyEval_EvalFrameEx ()
#26 0x0000000000575d92 in PyEval_EvalCodeEx ()
#27 0x00000000004c1352 in PyRun_SimpleFileExFlags ()
#28 0x00000000004c754f in Py_Main ()
#29 0x00007ffff68cb76d in __libc_start_main (main=0x41ba10 <main>, argc=2,
ubp_av=0x7fffffffdd28, init=<optimized out>, fini=<optimized out>,
rtld_fini=<optimized out>, stack_end=0x7fffffffdd18) at libc-start.c:226
#30 0x000000000041ba41 in _start ()