irqflags.h 4.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210
  1. #ifndef _X86_IRQFLAGS_H_
  2. #define _X86_IRQFLAGS_H_
  3. #include <asm/processor-flags.h>
  4. #ifndef __ASSEMBLY__
  5. /*
  6. * Interrupt control:
  7. */
  8. static inline unsigned long native_save_fl(void)
  9. {
  10. unsigned long flags;
  11. /*
  12. * "=rm" is safe here, because "pop" adjusts the stack before
  13. * it evaluates its effective address -- this is part of the
  14. * documented behavior of the "pop" instruction.
  15. */
  16. asm volatile("# __raw_save_flags\n\t"
  17. "pushf ; pop %0"
  18. : "=rm" (flags)
  19. : /* no input */
  20. : "memory");
  21. return flags;
  22. }
  23. static inline void native_restore_fl(unsigned long flags)
  24. {
  25. asm volatile("push %0 ; popf"
  26. : /* no output */
  27. :"g" (flags)
  28. :"memory", "cc");
  29. }
  30. static inline void native_irq_disable(void)
  31. {
  32. asm volatile("cli": : :"memory");
  33. }
  34. static inline void native_irq_enable(void)
  35. {
  36. asm volatile("sti": : :"memory");
  37. }
  38. static inline void native_safe_halt(void)
  39. {
  40. asm volatile("sti; hlt": : :"memory");
  41. }
  42. static inline void native_halt(void)
  43. {
  44. asm volatile("hlt": : :"memory");
  45. }
  46. #endif
  47. #ifdef CONFIG_PARAVIRT
  48. #include <asm/paravirt.h>
  49. #else
  50. #ifndef __ASSEMBLY__
  51. #include <linux/types.h>
  52. static inline notrace unsigned long arch_local_save_flags(void)
  53. {
  54. return native_save_fl();
  55. }
  56. static inline notrace void arch_local_irq_restore(unsigned long flags)
  57. {
  58. native_restore_fl(flags);
  59. }
  60. static inline notrace void arch_local_irq_disable(void)
  61. {
  62. native_irq_disable();
  63. }
  64. static inline notrace void arch_local_irq_enable(void)
  65. {
  66. native_irq_enable();
  67. }
  68. /*
  69. * Used in the idle loop; sti takes one instruction cycle
  70. * to complete:
  71. */
  72. static inline void arch_safe_halt(void)
  73. {
  74. native_safe_halt();
  75. }
  76. /*
  77. * Used when interrupts are already enabled or to
  78. * shutdown the processor:
  79. */
  80. static inline void halt(void)
  81. {
  82. native_halt();
  83. }
  84. /*
  85. * For spinlocks, etc:
  86. */
  87. static inline notrace unsigned long arch_local_irq_save(void)
  88. {
  89. unsigned long flags = arch_local_save_flags();
  90. arch_local_irq_disable();
  91. return flags;
  92. }
  93. #else
  94. #define ENABLE_INTERRUPTS(x) sti
  95. #define DISABLE_INTERRUPTS(x) cli
  96. #ifdef CONFIG_X86_64
  97. #define SWAPGS swapgs
  98. /*
  99. * Currently paravirt can't handle swapgs nicely when we
  100. * don't have a stack we can rely on (such as a user space
  101. * stack). So we either find a way around these or just fault
  102. * and emulate if a guest tries to call swapgs directly.
  103. *
  104. * Either way, this is a good way to document that we don't
  105. * have a reliable stack. x86_64 only.
  106. */
  107. #define SWAPGS_UNSAFE_STACK swapgs
  108. #define PARAVIRT_ADJUST_EXCEPTION_FRAME /* */
  109. #define INTERRUPT_RETURN jmp native_iret
  110. #define USERGS_SYSRET64 \
  111. swapgs; \
  112. sysretq;
  113. #define USERGS_SYSRET32 \
  114. swapgs; \
  115. sysretl
  116. #define ENABLE_INTERRUPTS_SYSEXIT32 \
  117. swapgs; \
  118. sti; \
  119. sysexit
  120. #else
  121. #define INTERRUPT_RETURN iret
  122. #define ENABLE_INTERRUPTS_SYSEXIT sti; sysexit
  123. #define GET_CR0_INTO_EAX movl %cr0, %eax
  124. #endif
  125. #endif /* __ASSEMBLY__ */
  126. #endif /* CONFIG_PARAVIRT */
  127. #ifndef __ASSEMBLY__
  128. static inline int arch_irqs_disabled_flags(unsigned long flags)
  129. {
  130. return !(flags & X86_EFLAGS_IF);
  131. }
  132. static inline int arch_irqs_disabled(void)
  133. {
  134. unsigned long flags = arch_local_save_flags();
  135. return arch_irqs_disabled_flags(flags);
  136. }
  137. #else
  138. #ifdef CONFIG_X86_64
  139. #define ARCH_LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk
  140. #define ARCH_LOCKDEP_SYS_EXIT_IRQ \
  141. TRACE_IRQS_ON; \
  142. sti; \
  143. SAVE_REST; \
  144. LOCKDEP_SYS_EXIT; \
  145. RESTORE_REST; \
  146. cli; \
  147. TRACE_IRQS_OFF;
  148. #else
  149. #define ARCH_LOCKDEP_SYS_EXIT \
  150. pushl %eax; \
  151. pushl %ecx; \
  152. pushl %edx; \
  153. call lockdep_sys_exit; \
  154. popl %edx; \
  155. popl %ecx; \
  156. popl %eax;
  157. #define ARCH_LOCKDEP_SYS_EXIT_IRQ
  158. #endif
  159. #ifdef CONFIG_TRACE_IRQFLAGS
  160. # define TRACE_IRQS_ON call trace_hardirqs_on_thunk;
  161. # define TRACE_IRQS_OFF call trace_hardirqs_off_thunk;
  162. #else
  163. # define TRACE_IRQS_ON
  164. # define TRACE_IRQS_OFF
  165. #endif
  166. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  167. # define LOCKDEP_SYS_EXIT ARCH_LOCKDEP_SYS_EXIT
  168. # define LOCKDEP_SYS_EXIT_IRQ ARCH_LOCKDEP_SYS_EXIT_IRQ
  169. # else
  170. # define LOCKDEP_SYS_EXIT
  171. # define LOCKDEP_SYS_EXIT_IRQ
  172. # endif
  173. #endif /* __ASSEMBLY__ */
  174. #endif