apache无法启动报错No space left on device

故障现象:apache无法启动ipcs信号量很多

# service httpd start
Starting httpd : [Failed]

当达到极限信号量：

# ipcs -s | wc -l
32004

而：

# cat /proc/sys/kernel/msgmni
32000

# cat /proc/sys/kernel/sem
250 24000 32 1024

原因
这些错误意味着有缺乏在系统内处理的通信资源，例如信号量或共享存储器段。

解析度
登录到通过服务器的SSH。

增加的限制sysctl 配置：

# mkdir -p /etc/sysctl.d/
# touch /etc/sysctl.d/99-zz_plesk_semaphores.conf
# chmod 755 /etc/sysctl.d/ /etc/sysctl.d/99-zz_plesk_semaphores.conf
# chown root:root /etc/sysctl.d/ /etc/sysctl.d/99-zz_plesk_semaphores.conf

这样的 /etc/sysctl.d/99-zz_plesk_semaphores.conf配置文件必须包含：

kernel.msgmni = 64000
kernel.sem = 250 256000 32 1024

负荷了新的sysctl设置 sysctl

# sysctl -p

检查哪些用户用尽信号灯：

# ipcs -s | awk '{print $3}' | uniq -c
4 root
33 httpd
234 somesoftware

阿帕奇保持清洁信号灯：

# for i in `ipcs -s | awk '/httpd/ {print $2}'`; do (ipcrm -s $i); done

在某些情况下，其他软件滥用严重限制信号，禁用该软件，清理作进一步调查其信号量和接触软件供应商：

# for i in `ipcs -s | awk '/httpd/ {print $2}'`; do (ipcrm -s $i); done

for i in `ipcs -s | awk '/httpd/{ next; } {print $2}'`; do (ipcrm -s $i); done

执行清理后问题依旧，反复执行也无效

[root@server:~]# ipcs -s

------ Semaphore Arrays --------
key semid owner perms nsems
0x00000000 0 root 600 1
0x00000000 65537 root 600 1
0x00000000 131074 apache 600 1
0x00000000 163843 apache 600 1
0x7a004a4e 196612 zabbix 600 13
0x00000000 229381 apache 600 1
0x00000000 262150 apache 600 1
0x00000000 294919 apache 600 1
0x00000000 327688 apache 600 1

[root@server:~]# ipcs -s | grep apache | perl -e 'while (<STDIN>) { @a=split(/\s+/); print `ipcrm sem $a[1]`}'

resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted
resource(s) deleted

for i in `ipcs -s | awk '/httpd/{ next; } {print $2}'`; do (ipcrm -s $i); done
[root@server:~]# ipcs -s

------ Semaphore Arrays --------
key semid owner perms nsems
0x00000000 0 root 600 1
0x00000000 65537 root 600 1
0x7a004a4e 196612 zabbix 600 13

日志：

[root@server:~]# tail -f /var/log/messages
Dec 4 04:47:36 server nrpe[36326]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:47:36 server nrpe[36328]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:47:36 server nrpe[36330]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:47:36 server nrpe[36332]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:47:36 server nrpe[36334]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:48:07 server nrpe[36711]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:48:07 server nrpe[36713]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:48:07 server nrpe[36715]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:49:04 server nrpe[37324]: Host 192.168.254.63 is not allowed to talk to us!
Dec 4 04:49:04 server nrpe[37333]: Host 192.168.254.63 is not allowed to talk to us!

# 原因是因为切分、监控等程序造成的，全部kill就可以了
[root@server:~]# ps -ef|grep apache|grep -v grep
root 18475 18474 0 16:50 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 18477 18475 0 16:50 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 26621 26618 0 17:00 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 26623 26621 0 17:00 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 34354 34352 0 17:10 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 34355 34354 0 17:10 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 41177 41175 0 17:20 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 41178 41177 0 17:20 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 47964 47962 0 17:30 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 47965 47964 0 17:30 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 49905 49904 0 14:20 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 49907 49905 0 14:20 ? 00:00:03 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 54555 54554 0 17:40 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 54557 54555 0 17:40 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 56774 56773 0 14:30 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 56775 56774 0 14:30 ? 00:00:03 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 61962 61960 0 17:50 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 61963 61962 0 17:50 ? 00:00:00 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 70159 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/error.log.%Y-%m-%d 86400
root 70160 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/i-job.chinasoft.com_error.log.%Y-%m-%d 86400
root 70161 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/support.chinasoft.com.old_error.log.%Y-%m-%d 86400
root 70162 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/error/dlcbs.chinasoft.com_error.log.%Y-%m-%d 86400
root 70163 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/access.log.%Y-%m-%d 86400
root 70164 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/i-job.chinasoft.com_access.log.%Y-%m-%d 86400
root 70165 70158 0 17:57 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/dlcbs.chinasoft.com_access.log.%Y-%m-%d 86400
apache 70166 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70167 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70168 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70169 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70170 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70171 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70172 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70173 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70174 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70175 70158 0 17:57 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
root 84551 84548 0 06:00 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 84553 84551 0 06:00 ? 00:00:23 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 97069 97068 0 15:30 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 97071 97069 0 15:30 ? 00:00:01 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 109607 109605 0 15:50 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 109608 109607 0 15:50 ? 00:00:01 /bin/bash /usr/local/worksh/monitor_apache_thread.sh
root 115643 115642 0 16:00 ? 00:00:00 /bin/sh -c /bin/bash /usr/local/worksh/monitor_apache_thread.sh > /dev/null 2>&1
root 115645 115643 0 16:00 ? 00:00:01 /bin/bash /usr/local/worksh/monitor_apache_thread.sh

[root@server:~]# ps -ef|grep apache|grep -v grep|awk '{print $2}'|xargs kill -9
[root:~]# ps -ef|grep apache
root 70590 70566 0 17:58 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/access.log.%Y-%m-%d 86400
root 70591 70566 0 17:58 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/i-job.chinasoft.com_access.log.%Y-%m-%d 86400
root 70592 70566 0 17:58 ? 00:00:00 /usr/local/apache2/bin/rotatelogs -l /data/www/logs/apache_log/access/dlcbs.chinasoft.com_access.log.%Y-%m-%d 86400
apache 70608 70566 0 17:58 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70609 70566 0 17:58 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
apache 70610 70566 0 17:58 ? 00:00:00 /usr/local/httpd-2.2.26/bin/httpd -k start
root 70612 35669 0 17:58 pts/0 00:00:00 grep apache

[root:~]# killall httpd

[root:~]# ipcs -s

------ Semaphore Arrays --------
key semid owner perms nsems

# 删除信号

for i in `ipcs -s | awk '/httpd/{ next; } {print $2}'`; do (ipcrm -s $i); done

最新文章

  1. &quot;Resuming debugger: error during debugging loop: TypeError: firstViewRangeElement is null&quot;
  2. 为AM335x移植Linux内核主线代码(35)使用platform中的GPIO
  3. Window Server 2008 R2 TFS2010的安装和配置
  4. java中异步多线程超时导致的服务异常
  5. Java 集合系列 10 Hashtable详细介绍(源码解析)和使用示例
  6. The Coco-Cola Store C(Contest #3 )
  7. ubuntu 11.10 安装apache2 tomcat6
  8. What Makes a Good Programmer Good?
  9. Ubuntu C++环境支持
  10. 20141015--for语句2
  11. 基于ArcGIS的栅格图像平滑处理(转)
  12. WPF中实现根据拼音查找汉字
  13. jdk8永久代从方法区移除的验证
  14. 关于springboot启动时候报错:springboot Failed to parse configuration class [Application]
  15. Java IO学习--(一)概述
  16. Faster R-CNN
  17. pointer-events: none 的两个应用场景
  18. P1140 相似基因 最长公共子序列
  19. D5 LCA 最近公共祖先
  20. Usefull Resources

热门文章

  1. SpringMVC 接受请求参数、作用域传值
  2. jsonp 实现前端跨域
  3. 买房安全无忧 l 龙光集团与光大银行二手房资金监管战略合作！
  4. Vjudge Code
  5. 报错utf-8错误
  6. 本部jdk切换的坑！！！
  7. Nginx+rtmp+ffmpeg 搭建推流服务器
  8. 洛谷P1169[ZJOI2007]棋盘制作
  9. Day057--django
  10. 金蝶K3 wise 插件二次开发与配置