Skywalking Php注册不上问题排查
Skywalking是一款分布式追踪应用,具体介绍可以参考 skywalking。
最近公司的一个Php应用在Skywalking后台查不到数据了:
登录到某台服务器上发现注册不上,启动时就报错了:
先来整理下Skywalking php的整个流程,php扩展在系统启动时注册应用和实例,然后在每次请求拦截相关调用,将相关调用情况保存下来;注册相关代码在skywalking.c的module_init中:
static void module_init() {
application_instance = -100000;
application_id = -100000;
int i = 0;
do {
application_id = serviceRegister(SKYWALKING_G(grpc), SKYWALKING_G(app_code));
if(application_id == -100000) {
sleep(1);
}
i++;
} while (application_id == -100000 && i <= 1);
if (application_id == -100000) {
sky_close = 1;
return;
}
char *ipv4s = _get_current_machine_ip();
char hostname[100] = {0};
if (gethostname(hostname, sizeof(hostname)) < 0) {
strcpy(hostname, "");
}
char *l_millisecond = get_millisecond();
long millisecond = zend_atol(l_millisecond, strlen(l_millisecond));
efree(l_millisecond);
i = 0;
do {
application_instance = serviceInstanceRegister(SKYWALKING_G(grpc), application_id, millisecond, SKY_OS_NAME,
hostname, getpid(),
ipv4s);
if(application_instance == -100000) {
sleep(2);
}
i++;
} while (application_instance == -100000 && i <= 3);
if (application_instance == -100000) {
sky_close = 1;
php_error(E_WARNING, "skywalking: register service error");
return;
}
php_error(E_WARNING, "skywalking: register service success");
}
可以看到,注册应用是调用serviceRegister函数注册,然后调用serviceInstanceRegister来注册实例的,后者会调用GreeterClient::serviceInstanceRegister以下函数完成注册:
int serviceInstanceRegister(int applicationid, long registertime, char *osname, char *hostname, int processno,
char *ipv4s) {
ServiceInstances request;
ServiceInstance *s = request.add_instances();
if (uuid == NULL) {
std::string uuid_str = boost::uuids::to_string(boost_uuid);
uuid = (char *) malloc(uuid_str.size() + 1);
bzero(uuid, uuid_str.size() + 1);
strncpy(uuid, uuid_str.c_str(), uuid_str.size() + 1);
}
s->set_serviceid(applicationid);
s->set_instanceuuid(std::string(uuid));
s->set_time(registertime);
KeyStringValuePair *os = s->add_properties();
KeyStringValuePair *host = s->add_properties();
KeyStringValuePair *process = s->add_properties();
KeyStringValuePair *ipv4 = s->add_properties();
KeyStringValuePair *language = s->add_properties();
os->set_key("os_name");
os->set_value(osname);
host->set_key("host_name");
host->set_value(hostname);
process->set_key("process_no");
process->set_value(std::to_string(processno));
ipv4->set_key("ipv4");
ipv4->set_value(ipv4s);
language->set_key("language");
language->set_value("php");
ServiceInstanceRegisterMapping reply;
ClientContext context;
Status status = stub_->doServiceInstanceRegister(&context, request, &reply);
if (status.ok()) {
for (int i = 0; i < reply.serviceinstances_size(); i++) {
const KeyIntValuePair &kv = reply.serviceinstances(i);
// std::cout << "Register Instance:"<< std::endl;
// std::cout << kv.key() << ": " << kv.value() << std::endl;
if (kv.key() == uuid) {
return kv.value();
}
}
}
return -100000;
}
通过gdb的断点,发现注册应用是成功的,注册实例失败了,然后在GreeterClient::serviceInstanceRegister加上相应的日志:
if (status.ok()) {
std::cout << "size:" << reply.serviceinstances_size() << std::endl;
for (int i = 0; i < reply.serviceinstances_size(); i++) {
const KeyIntValuePair &kv = reply.serviceinstances(i);
std::cout << "Register Instance:"<< std::endl;
std::cout << kv.key() << ": " << kv.value() << std::endl;
if (kv.key() == uuid) {
return kv.value();
}
}
}else{
printf("instance register error");
}
这里也把UUID打印出来了,发现响应里每次都是空的,所以导致失败。
客户端已经没有线索了,只好从服务端入手,因为服务端是Java实现的,不大方便调试,因此在本地搭了个环境想调试下,哪知服务端跑起来了,Php客户端死活编译不上,因为Skywalking依赖protobuf、grpc等组件,这些组件之间有版本依赖关系的,官方文档也没有说明,一时陷入困境。
因之前服务端维护的同学走了,只好自己硬着头皮看代码,发现注册入口代码在RegisterServiceHandler::doServiceInstanceRegister中:
@Override
public void doServiceInstanceRegister(ServiceInstances request,
StreamObserver<ServiceInstanceRegisterMapping> responseObserver) {
ServiceInstanceRegisterMapping.Builder builder = ServiceInstanceRegisterMapping.newBuilder();
request.getInstancesList().forEach(instance -> {
ServiceInventory serviceInventory = serviceInventoryCache.get(instance.getServiceId());
JsonObject instanceProperties = new JsonObject();
List<String> ipv4s = new ArrayList<>();
for (KeyStringValuePair property : instance.getPropertiesList()) {
String key = property.getKey();
switch (key) {
case HOST_NAME:
instanceProperties.addProperty(HOST_NAME, property.getValue());
break;
case OS_NAME:
instanceProperties.addProperty(OS_NAME, property.getValue());
break;
case LANGUAGE:
instanceProperties.addProperty(LANGUAGE, property.getValue());
break;
case "ipv4":
ipv4s.add(property.getValue());
break;
case PROCESS_NO:
instanceProperties.addProperty(PROCESS_NO, property.getValue());
break;
}
}
instanceProperties.addProperty(IPV4S, ServiceInstanceInventory.PropertyUtil.ipv4sSerialize(ipv4s));
String instanceName = serviceInventory.getName();
if (instanceProperties.has(PROCESS_NO)) {
instanceName += "-pid:" + instanceProperties.get(PROCESS_NO).getAsString();
}
if (instanceProperties.has(HOST_NAME)) {
instanceName += "@" + instanceProperties.get(HOST_NAME).getAsString();
}
int serviceInstanceId = serviceInstanceInventoryRegister.getOrCreate(instance.getServiceId(), instanceName, instance.getInstanceUUID(), instance.getTime(), instanceProperties);
if (serviceInstanceId != Const.NONE) {
logger.info("register service instance id={} [UUID:{}]", serviceInstanceId, instance.getInstanceUUID());
builder.addServiceInstances(KeyIntValuePair.newBuilder().setKey(instance.getInstanceUUID()).setValue(serviceInstanceId));
}
});
responseObserver.onNext(builder.build());
responseObserver.onCompleted();
}
关键是这行代码来生成实例id的:
int serviceInstanceId = serviceInstanceInventoryRegister.getOrCreate(instance.getServiceId(), instanceName, instance.getInstanceUUID(), instance.getTime(), instanceProperties);
再跟进去:
@Override public int getOrCreate(int serviceId, String serviceInstanceName, String uuid, long registerTime,
JsonObject properties) {
if (logger.isDebugEnabled()) {
logger.debug("Get or create service instance by service instance name, service id: {}, service instance name: {},uuid: {}, registerTime: {}", serviceId, serviceInstanceName, uuid, registerTime);
}
int serviceInstanceId = getServiceInstanceInventoryCache().getServiceInstanceId(serviceId, uuid);
if (serviceInstanceId == Const.NONE) {
ServiceInstanceInventory serviceInstanceInventory = new ServiceInstanceInventory();
serviceInstanceInventory.setServiceId(serviceId);
serviceInstanceInventory.setName(serviceInstanceName);
serviceInstanceInventory.setInstanceUUID(uuid);
serviceInstanceInventory.setIsAddress(BooleanUtils.FALSE);
serviceInstanceInventory.setAddressId(Const.NONE);
serviceInstanceInventory.setRegisterTime(registerTime);
serviceInstanceInventory.setHeartbeatTime(registerTime);
serviceInstanceInventory.setProperties(properties);
InventoryStreamProcessor.getInstance().in(serviceInstanceInventory);
}
return serviceInstanceId;
}
这里的逻辑就比较清晰了,先从缓存中拿实例ID:
getServiceInstanceInventoryCache().getServiceInstanceId(serviceId, uuid);
拿不到则加入后台任务处理生成ID。
再跟进getServiceInstanceId方法,
if (Objects.isNull(serviceInstanceId) || serviceInstanceId == Const.NONE) {
serviceInstanceId = getCacheDAO().getServiceInstanceId(serviceId, uuid);
if (serviceId != Const.NONE) {
serviceInstanceNameCache.put(ServiceInstanceInventory.buildId(serviceId, uuid), serviceInstanceId);
}
}
从缓存中拿不到则从DAO中拿,
GetResponse response = getClient().get(ServiceInstanceInventory.INDEX_NAME, id);
if (response.isExists()) {
return (int)response.getSource().getOrDefault(RegisterSource.SEQUENCE, 0);
} else {
return Const.NONE;
}
后者从ES索引service_instance_inventory去拿。
为了证实上述逻辑无误,从ES中读取数据试下,果然实例ID都注册在ES里面:
再从客户端证实下,既然实例ID是写入ES的,那么用以前的ID肯定是能注册成功的,因此修改客户端代码,将UUID写死注册试下:
int serviceInstanceRegister(int applicationid, long registertime, char *osname, char *hostname, int processno,
char *ipv4s) {
ServiceInstances request;
ServiceInstance *s = request.add_instances();
uuid= "7e22c317-e2e2-4f81-a53d-fe011013e0a3";
if (uuid == NULL) {
std::string uuid_str = boost::uuids::to_string(boost_uuid);
uuid = (char *) malloc(uuid_str.size() + 1);
bzero(uuid, uuid_str.size() + 1);
strncpy(uuid, uuid_str.c_str(), uuid_str.size() + 1);
}
马上注册成功了:
7e22c317-e2e2-4f81-a53d-fe011013e0a3
size:1
Register Instance:
7e22c317-e2e2-4f81-a53d-fe011013e0a3: 3386041
PHP Warning: skywalking: register service success in Unknown on line 0
PHP Warning: skywalking: hook redis handler success in Unknown on line 0
PHP Warning: skywalking: hook session handler success in Unknown on line 0
再回到这个问题,原因已经知道了,如何解决呢,有两个办法:
1、加大注册时等待时间,如等待到100秒;
2、记录最近一次注册成功的UUID并且持久化,下次启动时直接用上次的;
因为2涉及到改代码,因此先用方案1解决问题。
- SendCloud邮件队列状态和已使用额度的Python监控脚本
- linux/scp命令报“bash: scp: command not found lost connection”错误的解决办法
- bat/cmd批处理连接SqlServer数据库查询脚本
- 一起用 HTML5 Canvas 做一个简单又骚气的粒子引擎
- 解决mstsc无法连接问题:由于没有远程桌面授权服务器可以提供许可证…
- Apache/Nginx伪静态规则匹配http://出现的问题与解决
- 微信文件微起底
- Go语言TCP Socket编程--1
- Go语言TCP Socket编程--2
- 服务器 数据库设计技巧--1
- CVE-2015-0235:Linux glibc高危漏洞的检测及修复方法
- zabbix监控在lnmp环境下编译安装小记
- 【重磅】百度开源分布式深度学习平台,挑战TensorFlow (教程)
- WordPress评论ajax动态加载,解决静态缓存下评论不更新问题
- php概述
- php教程
- php环境搭建
- PHP书写格式
- php变量
- php常量
- PHP注释
- php数组
- php字符串 string
- PHP整型 integer
- PHP浮点型 float
- php布尔型
- php数据类型之数组
- php数据类型之对象
- php数据类型之null
- php数据类型之间的转换
- php运算符
- php表达式
- PHP循环控制
- PHP流程控制
- php函数
- php全局变量
- PHP魔术变量
- php命名空间
- php 日期
- PHP包含文件
- php文件
- PHP 文件上传
- php Cookies
- php Sessions
- php email
- php安全email
- php错误处理
- PHP异常处理
- php过滤器
- PHP 高级过滤器
- php json
- php 表单
- PHP MySQL 简介
- PHP 连接 MySQL
- php创建数据库
- php 创建表
- php mysq 插入数据
- PHP MySQL 插入多条数据
- PHP MySQL 预处理语句
- php mysql 读取数据
- php mysql where
- PHP MySQL Order By
- PHP MySQL Update
- PHP MySQL Delete
- php ODBC
- Android 带伸缩动画的布局
- iOS 图像处理技术追踪-Core Image
- Android BottomSheetDialog使用实现底部拖动弹窗
- Android 滑动渐变背景Toolbar、点击置顶ScrollView
- Android 自定义View 画圆(奥运五环)
- Android 触摸屏交互之手势监听
- Android Glide加载网络图片不显示,但用网页打开又正常显示
- Android获取APP的版本号和版本名
- Android 在一个APP里打开另一个APP
- Android 自定义加载动画Dialog弹窗
- Android 天气APP(十一)未来七天的天气预报、逐小时预报、UI优化
- Android 天气APP(十)继续优化、下拉刷新页面天气数据
- Activity 活动跳转(Java&Kotlin)
- Android 天气APP(九)细节优化、必应每日一图
- Android CheckBox修改选中颜色并去除选中时的水波纹效果