blkio.go 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294
  1. package cgroup
  2. import (
  3. "bufio"
  4. "os"
  5. "path/filepath"
  6. "strconv"
  7. "strings"
  8. "unicode"
  9. )
  10. // BlockIOSubsystem contains limits and metrics from the "blkio" subsystem. The
  11. // blkio subsystem controls and monitors access to I/O on block devices by tasks
  12. // in a cgroup.
  13. //
  14. // https://www.kernel.org/doc/Documentation/cgroup-v1/blkio-controller.txt
  15. type BlockIOSubsystem struct {
  16. Metadata
  17. Throttle ThrottlePolicy `json:"throttle,omitempty"` // Throttle limits for upper IO rates and metrics.
  18. //CFQ CFQScheduler `json:"cfq,omitempty"` // Completely fair queue scheduler limits and metrics.
  19. }
  20. // CFQScheduler contains limits and metrics for the proportional weight time
  21. // based division of disk policy. It is implemented in CFQ. Hence this policy
  22. // takes effect only on leaf nodes when CFQ is being used.
  23. //
  24. // https://www.kernel.org/doc/Documentation/block/cfq-iosched.txt
  25. type CFQScheduler struct {
  26. Weight uint64 `json:"weight"` // Default weight for all devices unless overridden. Allowed range of weights is from 10 to 1000.
  27. Devices []CFQDevice `json:"devices,omitempty"`
  28. }
  29. // CFQDevice contains CFQ limits and metrics associated with a single device.
  30. type CFQDevice struct {
  31. DeviceID DeviceID `json:"device_id"` // ID of the device.
  32. // Proportional weight for the device. 0 means a per device weight is not set and
  33. // that the blkio.weight value is used.
  34. Weight uint64 `json:"weight"`
  35. TimeMs uint64 `json:"time_ms"` // Disk time allocated to cgroup per device in milliseconds.
  36. Sectors uint64 `json:"sectors"` // Number of sectors transferred to/from disk by the cgroup.
  37. Bytes OperationValues `json:"io_service_bytes"` // Number of bytes transferred to/from the disk by the cgroup.
  38. IOs OperationValues `json:"io_serviced"` // Number of IO operations issued to the disk by the cgroup.
  39. ServiceTimeNanos OperationValues `json:"io_service_time"` // Amount of time between request dispatch and request completion for the IOs done by this cgroup.
  40. WaitTimeNanos OperationValues `json:"io_wait_time"` // Amount of time the IOs for this cgroup spent waiting in the scheduler queues for service.
  41. Merges OperationValues `json:"io_merged"` // Total number of bios/requests merged into requests belonging to this cgroup.
  42. }
  43. // ThrottlePolicy contains the upper IO limits and metrics for devices used
  44. // by the cgroup.
  45. type ThrottlePolicy struct {
  46. Devices []ThrottleDevice `json:"devices,omitempty"` // Device centric view of limits and metrics.
  47. TotalBytes uint64 `json:"total_io_service_bytes"` // Total number of bytes serviced by all devices.
  48. TotalIOs uint64 `json:"total_io_serviced"` // Total number of IO operations serviced by all devices.
  49. }
  50. // ThrottleDevice contains throttle limits and metrics associated with a single device.
  51. type ThrottleDevice struct {
  52. DeviceID DeviceID `json:"device_id"` // ID of the device.
  53. ReadLimitBPS uint64 `json:"read_bps_device"` // Read limit in bytes per second (BPS). Zero means no limit.
  54. WriteLimitBPS uint64 `json:"write_bps_device"` // Write limit in bytes per second (BPS). Zero mean no limit.
  55. ReadLimitIOPS uint64 `json:"read_iops_device"` // Read limit in IOPS. Zero means no limit.
  56. WriteLimitIOPS uint64 `json:"write_iops_device"` // Write limit in IOPS. Zero means no limit.
  57. Bytes OperationValues `json:"io_service_bytes"` // Number of bytes transferred to/from the disk by the cgroup.
  58. IOs OperationValues `json:"io_serviced"` // Number of IO operations issued to the disk by the cgroup.
  59. }
  60. // OperationValues contains the I/O limits or metrics associated with read,
  61. // write, sync, and async operations.
  62. type OperationValues struct {
  63. Read uint64 `json:"read"`
  64. Write uint64 `json:"write"`
  65. Async uint64 `json:"async"`
  66. Sync uint64 `json:"sync"`
  67. }
  68. // DeviceID identifies a Linux block device.
  69. type DeviceID struct {
  70. Major uint64
  71. Minor uint64
  72. }
  73. // blkioValue holds a single blkio value associated with a device.
  74. type blkioValue struct {
  75. DeviceID
  76. Operation string
  77. Value uint64
  78. }
  79. // get reads metrics from the "blkio" subsystem. path is the filepath to the
  80. // cgroup hierarchy to read.
  81. func (blkio *BlockIOSubsystem) get(path string) error {
  82. if err := blkioThrottle(path, blkio); err != nil {
  83. return err
  84. }
  85. // TODO(akroh): Implement reading for the CFQ values.
  86. return nil
  87. }
  88. // blkioThrottle reads all of the limits and metrics associated with blkio
  89. // throttling policy.
  90. func blkioThrottle(path string, blkio *BlockIOSubsystem) error {
  91. devices := map[DeviceID]*ThrottleDevice{}
  92. getDevice := func(id DeviceID) *ThrottleDevice {
  93. td := devices[id]
  94. if td == nil {
  95. td = &ThrottleDevice{DeviceID: id}
  96. devices[id] = td
  97. }
  98. return td
  99. }
  100. values, err := readBlkioValues(path, "blkio.throttle.io_service_bytes")
  101. if err != nil {
  102. return err
  103. }
  104. if values != nil {
  105. for id, opValues := range collectOpValues(values) {
  106. getDevice(id).Bytes = *opValues
  107. }
  108. }
  109. values, err = readBlkioValues(path, "blkio.throttle.io_serviced")
  110. if err != nil {
  111. return err
  112. }
  113. if values != nil {
  114. for id, opValues := range collectOpValues(values) {
  115. getDevice(id).IOs = *opValues
  116. }
  117. }
  118. values, err = readBlkioValues(path, "blkio.throttle.read_bps_device")
  119. if err != nil {
  120. return err
  121. }
  122. if values != nil {
  123. for _, bv := range values {
  124. getDevice(bv.DeviceID).ReadLimitBPS = bv.Value
  125. }
  126. }
  127. values, err = readBlkioValues(path, "blkio.throttle.write_bps_device")
  128. if err != nil {
  129. return err
  130. }
  131. if values != nil {
  132. for _, bv := range values {
  133. getDevice(bv.DeviceID).WriteLimitBPS = bv.Value
  134. }
  135. }
  136. values, err = readBlkioValues(path, "blkio.throttle.read_iops_device")
  137. if err != nil {
  138. return err
  139. }
  140. if values != nil {
  141. for _, bv := range values {
  142. getDevice(bv.DeviceID).ReadLimitIOPS = bv.Value
  143. }
  144. }
  145. values, err = readBlkioValues(path, "blkio.throttle.write_iops_device")
  146. if err != nil {
  147. return err
  148. }
  149. if values != nil {
  150. for _, bv := range values {
  151. getDevice(bv.DeviceID).WriteLimitIOPS = bv.Value
  152. }
  153. }
  154. blkio.Throttle.Devices = make([]ThrottleDevice, 0, len(devices))
  155. for _, dev := range devices {
  156. blkio.Throttle.Devices = append(blkio.Throttle.Devices, *dev)
  157. blkio.Throttle.TotalBytes += dev.Bytes.Read + dev.Bytes.Write
  158. blkio.Throttle.TotalIOs += dev.IOs.Read + dev.IOs.Write
  159. }
  160. return nil
  161. }
  162. // collectOpValues collects the discreet I/O values (e.g. read, write, sync,
  163. // async) for a given device into a single OperationValues object. It returns a
  164. // mapping of device ID to OperationValues.
  165. func collectOpValues(values []blkioValue) map[DeviceID]*OperationValues {
  166. opValues := map[DeviceID]*OperationValues{}
  167. for _, bv := range values {
  168. opValue := opValues[bv.DeviceID]
  169. if opValue == nil {
  170. opValue = &OperationValues{}
  171. opValues[bv.DeviceID] = opValue
  172. }
  173. switch bv.Operation {
  174. case "read":
  175. opValue.Read = bv.Value
  176. case "write":
  177. opValue.Write = bv.Value
  178. case "async":
  179. opValue.Async = bv.Value
  180. case "sync":
  181. opValue.Sync = bv.Value
  182. }
  183. }
  184. return opValues
  185. }
  186. // readDeviceValues reads values from a single blkio file.
  187. // It expects to read values like "245:1 read 18880" or "254:1 1909". It returns
  188. // an array containing an entry for each valid line read.
  189. func readBlkioValues(path ...string) ([]blkioValue, error) {
  190. f, err := os.Open(filepath.Join(path...))
  191. if err != nil {
  192. if os.IsNotExist(err) {
  193. return nil, nil
  194. }
  195. return nil, err
  196. }
  197. defer f.Close()
  198. var values []blkioValue
  199. sc := bufio.NewScanner(f)
  200. for sc.Scan() {
  201. line := strings.TrimSpace(sc.Text())
  202. if len(line) == 0 {
  203. continue
  204. }
  205. // Valid lines start with a device ID.
  206. if !unicode.IsNumber(rune(line[0])) {
  207. continue
  208. }
  209. v, err := parseBlkioValue(sc.Text())
  210. if err != nil {
  211. return nil, err
  212. }
  213. values = append(values, v)
  214. }
  215. return values, sc.Err()
  216. }
  217. func isColonOrSpace(r rune) bool {
  218. return unicode.IsSpace(r) || r == ':'
  219. }
  220. func parseBlkioValue(line string) (blkioValue, error) {
  221. fields := strings.FieldsFunc(line, isColonOrSpace)
  222. if len(fields) != 3 && len(fields) != 4 {
  223. return blkioValue{}, ErrInvalidFormat
  224. }
  225. major, err := strconv.ParseUint(fields[0], 10, 64)
  226. if err != nil {
  227. return blkioValue{}, err
  228. }
  229. minor, err := strconv.ParseUint(fields[1], 10, 64)
  230. if err != nil {
  231. return blkioValue{}, err
  232. }
  233. var value uint64
  234. var operation string
  235. if len(fields) == 3 {
  236. value, err = parseUint([]byte(fields[2]))
  237. if err != nil {
  238. return blkioValue{}, err
  239. }
  240. } else {
  241. operation = strings.ToLower(fields[2])
  242. value, err = parseUint([]byte(fields[3]))
  243. if err != nil {
  244. return blkioValue{}, err
  245. }
  246. }
  247. return blkioValue{
  248. DeviceID: DeviceID{major, minor},
  249. Operation: operation,
  250. Value: value,
  251. }, nil
  252. }