zookeeper 설치

# zookeeper 설치 파일 다운로드 
[kalva@kalva-zk01 ~]$ wget https://dlcdn.apache.org/zookeeper/zookeeper-3.8.1/apache-zookeeper-3.8.1-bin.tar.gz

# zookeeper 설치 파일 압축해제 
[kalva@kalva-zk01 ~]$ sudo tar xvzf apache-zookeeper-3.8.1-bin.tar.gz -C /usr/local/

# zookeeper 설치파일 /usr/local/zookeeper 로 경로 설정 
[kalva@kalva-zk01 ~]$ cd /usr/local
[kalva@kalva-zk01 local]$ sudo ln -s ./apache-zookeeper-3.8.1-bin ./zookeeper

# zookeeper logs 디렉토리 생성 
[kalva@kalva-zk01 local]$ sudo mkdir -p /usr/local/zookeeper/logs

# zookeeper 데이터 디렉토리 생성 
[kalva@kalva-zk01 local]$ mkdir -p /data/zk

# zookeeper 구분 id 파일 생성 (각 서버에 맞게 생성)
## kalva-zk01 : 1
## kalva-zk02 : 2
## kalva-zk03 : 3
[kalva@kalva-zk01 local]$ echo 1 | sudo tee /data/zk/myid

# zookeeper Configuration 파일 설정
[kalva@kalva-zk01 local]$ echo "tickTime=2000
initLimit=10
syncLimit=5
dataDir=/data/zk
clientPort=2181
autopurge.snapRetainCount=3
autopurge.purgeInterval=1
server.1=kalva-zk01:2888:3888;2181
server.2=kalva-zk02:2888:3888;2181
server.3=kalva-zk03:2888:3888;2181" | sudo tee /usr/local/zookeeper/conf/zoo.cfg

# systemctl zookeeper-server.service 파일 생성 
[kalva@kalva-zk01 local]$ echo "[Unit]
Description=zookeeper-server
After=network.target

[Service]
Type=forking
User=zookeeper
Group=zookeeper
SyslogIdentifier=zookeeper-server
WorkingDirectory=/usr/local/zookeeper
Restart=always
RestartSec=0s
ExecStart=/usr/local/zookeeper/bin/zkServer.sh start
ExecStop=/usr/local/zookeeper/bin/zkServer.sh stop
ExecReload=/usr/local/zookeeper/bin/zkServer.sh restart

[Install]
WantedBy=multi-user.target" | sudo tee /etc/systemd/system/zookeeper-server.service

# systemctl daemon-reload, zookeeper-server 사용설정 
[kalva@kalva-zk01 local]$ sudo systemctl daemon-reload
[kalva@kalva-zk01 local]$ sudo systemctl enable zookeeper-server

# zookeeper 디렉토리 owner 변경 
[kalva@kalva-zk01 local]$ sudo chown -R zookeeper:zookeeper /usr/local/apache-zookeeper-3.8.1-bin
[kalva@kalva-zk01 local]$ sudo chown -R zookeeper:zookeeper /usr/local/zookeeper
[kalva@kalva-zk01 local]$ sudo chown -R zookeeper:zookeeper /data

# zookeeper-server 구동, 상태 확인 
[kalva@kalva-zk01 local]$ sudo systemctl start zookeeper-server
[kalva@kalva-zk01 local]$ sudo systemctl status zookeeper-server
● zookeeper-server.service - zookeeper-server
   Loaded: loaded (/etc/systemd/system/zookeeper-server.service; enabled; vendor preset: disabled)
   Active: active (running) since Fri 2023-03-17 00:18:57 KST; 37s ago
  Process: 2264 ExecStart=/usr/local/zookeeper/bin/zkServer.sh start (code=exited, status=0/SUCCESS)
 Main PID: 2281 (java)
   CGroup: /system.slice/zookeeper-server.service
           └─2281 java -Dzookeeper.log.dir=/usr/local/zookeeper/bin/../logs -Dzookeeper.log.file=zookeeper-zookeeper-server-kalva-zk01.log -XX:+HeapDumpOnOutOfMemoryError -XX:OnOutOfMemoryEr...

Mar 17 00:18:56 kalva-zk01 systemd[1]: Starting zookeeper-server...
Mar 17 00:18:56 kalva-zk01 zookeeper-server[2264]: /usr/bin/java
Mar 17 00:18:56 kalva-zk01 zookeeper-server[2264]: ZooKeeper JMX enabled by default
Mar 17 00:18:56 kalva-zk01 zookeeper-server[2264]: Using config: /usr/local/zookeeper/bin/../conf/zoo.cfg
Mar 17 00:18:57 kalva-zk01 systemd[1]: Started zookeeper-server.

kafka 설치

# kafka 설치파일 다운로드 
[kalva@kalva-kafka01 ~]$ wget https://downloads.apache.org/kafka/3.4.0/kafka_2.13-3.4.0.tgz

# kafka 설치파일 압축해제
[kalva@kalva-kafka01 ~]$ tar xzf kafka_2.13-3.4.0.tgz

# kafka 디렉토리 설정 
[kalva@kalva-kafka01 ~]$ sudo mv kafka_2.13-3.4.0 /usr/local/
[kalva@kalva-kafka01 ~]$ sudo ln -s /usr/local/kafka_2.13-3.4.0 /usr/local/kafka

# kafka Configuration 파일 설정 (각 서버에 맞게 설정)
## kalva-kafka01 : broker.id=1, PLAINTEXT://kalva-kafka01:9092
## kalva-kafka02 : broker.id=3, PLAINTEXT://kalva-kafka02:9092
## kalva-kafka03 : broker.id=3, PLAINTEXT://kalva-kafka03:9092
[kalva@kalva-kafka01 ~]$ echo "broker.id=1
listeners=PLAINTEXT://0.0.0.0:9092
advertised.listeners=PLAINTEXT://kalva-kafka01:9092
num.network.threads=4
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/data/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=3
transaction.state.log.replication.factor=3
transaction.state.log.min.isr=2
log.retention.hours=72
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=kalva-zk01:2181,kalva-zk02:2181,kalva-zk03:2181
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=3000" | sudo tee /usr/local/kafka/config/server.properties

# jmx 설정 파일 설정 
[kalva@kalva-kafka01 ~]$ echo "JMX_PORT=9999" | sudo tee /usr/local/kafka/config/jmx

# systemctl kafka-server.service 파일 생성 
[kalva@kalva-kafka01 ~]$ echo "[Unit]
Description=kafka-server
After=network.target

[Service]
Type=simple
SyslogIdentifier=kafka-server
WorkingDirectory=/usr/local/kafka
EnvironmentFile=/usr/local/kafka/config/jmx
Restart=always
ExecStart=/usr/local/kafka/bin/kafka-server-start.sh /usr/local/kafka/config/server.properties
ExecStop=/usr/local/kafka/bin/kafka-server-stop.sh

[Install]
WantedBy=multi-user.target" | sudo tee /etc/systemd/system/kafka-server.service

# systemctl daemon-reload, kafka-server 사용설정 
[kalva@kalva-kafka01 ~]$ sudo systemctl daemon-reload
[kalva@kalva-kafka01 ~]$ sudo systemctl enable kafka-server

# kafka-server 구동, 상태 확인 
[kalva@kalva-kafka01 ~]$ sudo systemctl start kafka-server
[kalva@kalva-kafka01 ~]$ sudo systemctl status kafka-server
● kafka-server.service - kafka-server
   Loaded: loaded (/etc/systemd/system/kafka-server.service; enabled; vendor preset: disabled)
   Active: active (running) since Fri 2023-03-17 21:51:39 KST; 1s ago
 Main PID: 15487 (java)
   CGroup: /system.slice/kafka-server.service
           └─15487 java -Xmx1G -Xms1G -server -XX:+UseG1GC -XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:+ExplicitGCInvokesConcurrent -XX:MaxInlineLevel=15 -Djava.awt.headless=tr...

Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: at kafka.server.KafkaServer.startup(KafkaServer.scala:234)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: at kafka.Kafka$.main(Kafka.scala:115)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: at kafka.Kafka.main(Kafka.scala)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: [2023-03-17 21:51:41,575] INFO shutting down (kafka.server.KafkaServer)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: [2023-03-17 21:51:41,577] INFO [ZooKeeperClient Kafka server] Closing. (kafka.zookeeper.ZooKeeperClient)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: [2023-03-17 21:51:41,597] WARN An exception was thrown while closing send thread for session 0x300000075ce0002. (org.apache.zoo....ClientCnxn)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: EndOfStreamException: Unable to read additional data from server sessionid 0x300000075ce0002, likely server has closed socket
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: at org.apache.zookeeper.ClientCnxnSocketNIO.doIO(ClientCnxnSocketNIO.java:77)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350)
Mar 17 21:51:41 kalva-kafka01 kafka-server[15487]: at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1290)
Hint: Some lines were ellipsized, use -l to show in full.

# 토픽 생성 
[kalva@kalva-kafka01 ~]$ /usr/local/kafka/bin/kafka-topics.sh --bootstrap-server kalva-kafka01:9092 --replication-factor 1 --partitions 1 --create --topic test01
Created topic test01.

# 메시지 전송 
[kalva@kalva-kafka01 ~]$ /usr/local/kafka/bin/kafka-console-producer.sh --broker-list kalva-kafka01:9092,kalva-kafka02:9092,kalva-kafka03:9092 --topic test01
>hi! hi! hi!

# 메시지 읽어오기 
[kalva@kalva-kafka01 ~]$ /usr/local/kafka/bin/kafka-console-consumer.sh --bootstrap-server kalva-kafka01:9092 --topic test01 --from-beginning
hi! hi! hi!

'KAFKA' 카테고리의 다른 글

kminion 설치 (docker)  (0) 2023.06.22

Full Text Catalog 의 변경 내용 추적을 AUTO 로 운영중인데 서버 재기동 이후 채우기가 동작하지 않는 상황 발생 하였다.

 

검색해서 찾은 해결 방법은 

1. rebuild

2. 변경 내용 추적을 MANUAL 로 변경했다 AUTO 로 변경

3. 서버 재기동 

 

rebuild 는 시간 및 부하 이슈가 있으니 변경 내용 추적을 MANUAL 로 변경했다 AUTO 로 변경했다.

이후 주기적으로 잘 수집되는것을 확인하였다.

# Full Text Catalog 상태 확인 
declare @id int
select @id = id FROM sys.sysobjects where [Name] = 'TableName'
select 'TableFullTextBackgroundUpdateIndexOn' as 'Property', objectpropertyex(@id, 'TableFullTextBackgroundUpdateIndexOn') as 'Value'
union 
select 'TableFullTextChangeTrackingOn', objectpropertyex(@id, 'TableFullTextChangeTrackingOn')
union 
select 'TableFulltextDocsProcessed', objectpropertyex(@id, 'TableFulltextDocsProcessed') 
union 
select 'TableFulltextFailCount', objectpropertyex(@id, 'TableFulltextFailCount') 
union 
select 'TableFulltextItemCount', objectpropertyex(@id, 'TableFulltextItemCount') 
union 
select 'TableFulltextKeyColumn', objectpropertyex(@id, 'TableFulltextKeyColumn') 
union 
select 'TableFulltextPendingChanges', objectpropertyex(@id, 'TableFulltextPendingChanges') 
union 
select 'TableHasActiveFulltextIndex', objectpropertyex(@id, 'TableHasActiveFulltextIndex') 
union 
select 'TableFulltextPopulateStatus', objectpropertyex(@id, 'TableFulltextPopulateStatus') 

# 아래 쿼리의 Command 에서 생성된 쿼리로 변경 내용 추적을 MANUAL 로 변경했다 AUTO 로 변경한다.
# 주의할 점은 crawl_end_date가 없는 경우 전체를 다시 수집한다. 
SELECT [t].[name] [table_name], 
       [i].[name] [index_name],
       [fi].[change_tracking_state_desc], 
       [fi].[has_crawl_completed], 
       [fi].[crawl_type_desc], 
       [fi].[crawl_end_date],
       [ius].[last_user_update], 
       [ius].[last_user_seek],
       (SELECT [name]+',' FROM [sys].[fulltext_index_columns] [fc] INNER JOIN [sys].[columns] [c] ON [c].[object_id] = [fc].[object_id] AND [c].[column_id] = [fc].[column_id] WHERE [fc].[object_id] = [fi].[object_id] FOR XML PATH('')) [columns],
       (CASE WHEN [fi].[crawl_end_date] < ISNULL([ius].[last_user_update], [ius].[last_user_seek]) THEN 'ALTER FULLTEXT INDEX ON ['+[t].[name]+'] SET CHANGE_TRACKING MANUAL; ALTER FULLTEXT INDEX ON ['+[t].[name]+'] SET CHANGE_TRACKING AUTO' ELSE '' END) [Command]
FROM [sys].[fulltext_indexes] [fi]
INNER JOIN [sys].[indexes] [i] ON [i].[index_id] = [fi].[unique_index_id] AND [i].[object_id] = [fi].[object_id]
INNER JOIN [sys].[tables] [t] ON [t].[object_id] = [fi].[object_id]
LEFT JOIN [sys].[dm_db_index_usage_stats] [ius] ON [ius].[index_id] = [fi].[unique_index_id] AND [ius].[object_id] = [fi].[object_id] AND [ius].[database_id] = DB_ID()
ORDER BY [table_name], [index_name]

 

 

참고 : 

https://stackoverflow.com/questions/36329633/updating-fulltextsearch-index-immediately-sql-server-2012

https://www.mssqltips.com/sqlservertip/5996/tsql-script-to-correct-sql-server-full-text-indexes-not-updating/

'SQL Server' 카테고리의 다른 글

Always Encrypted char, nchar 데이터 타입 공백 검색 이슈  (0) 2022.09.02
Always Encrypted  (0) 2022.09.02
set ansi_padding off  (0) 2022.09.02

MySQL 서버의 CPU를 알뜰하게 다 사용해서 사용률이 100% 를 친다면?

MySQL은 쓰레드로 동작한다. CPU 사용률이 가장 높은 쓰레드를 찾아서 performance_schema.threads  에서 THREAD_OS_ID로 조회할 수 있다. 

 

pidstat 를 사용해서 CPU 사용률이 높은 쓰레드를 찾아본다. 

pidstat -t -p <mysqld_pid> 1  # 1 은 1초 간격으로 실행 

-- mysqld 의 pid 확인 
# ps -ef |grep mysqld
mysql        955       1  0 22:21 ?        00:00:00 /usr/sbin/mysqld --daemonize --pid-file=/var/run/mysqld/mysqld.pid
root        1622    1573  0 22:22 pts/1    00:00:00 grep --color=auto mysqld

# pidstat -t -p 955 1
-bash: pidstat: command not found

-- pidstat가 없을 경우 sysstat를 설치한다. 
# yum -y install sysstat

-- CPU 사용률이 가장 높은 TID 확인 
# pidstat -t -p 955 1
Linux 4.18.0-305.19.1.el8_4.x86_64 (localhost.localdomain) 	10/05/2022 	_x86_64_	(1 CPU)

10:24:46 PM   UID      TGID       TID    %usr %system  %guest   %wait    %CPU   CPU  Command
10:24:47 PM    27       955         -    1.98    0.99    0.00    0.00    2.97     0  mysqld
10:24:47 PM    27         -       955    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       959    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       963    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       965    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       966    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       967    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       968    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       969    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       970    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       971    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       972    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       973    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       974    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       994    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       995    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       996    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       997    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       998    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -       999    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -      1000    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -      1001    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -      1015    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -      1016    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -      1017    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -      1018    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -      1053    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -      1057    0.00    0.00    0.00    0.00    0.00     0  |__mysqld
10:24:47 PM    27         -      1564    2.97    0.99    0.00    0.99    3.96     0  |__mysqld

 

CPU 사용률이 가장 높은 TID 1565 번을 확인한다. 

-- THREAD_OS_ID = 1564 확인 
mysql> select * from performance_schema.threads where THREAD_OS_ID=1564\G;
*************************** 1. row ***************************
          THREAD_ID: 28
               NAME: thread/sql/one_connection
               TYPE: FOREGROUND
     PROCESSLIST_ID: 3
   PROCESSLIST_USER: root
   PROCESSLIST_HOST: localhost
     PROCESSLIST_DB: test
PROCESSLIST_COMMAND: Query
   PROCESSLIST_TIME: 101
  PROCESSLIST_STATE: query end
   PROCESSLIST_INFO: select * from tb_test where col1 = 'aaaa'
   PARENT_THREAD_ID: NULL
               ROLE: NULL
       INSTRUMENTED: YES
            HISTORY: YES
    CONNECTION_TYPE: Socket
       THREAD_OS_ID: 1564
1 row in set (0.00 sec)

ERROR: 
No query specified

-- PROCESSLIST_ID = 3 kill 
mysql> kill 3;
Query OK, 0 rows affected (0.01 sec)

'MySQL' 카테고리의 다른 글

xtrabackup 옵션  (1) 2023.11.30
lock session 찾기  (0) 2022.10.01
auto_increment 값 모니터링  (0) 2022.09.28
Query Cache  (0) 2022.09.24
Index Hint  (0) 2022.09.22

show engine innodb status 의 TRANSACTIONS 부분에서 ACTIVE TRANSACTION 으로 Lock 을 잡고 있는 쿼리를 찾아야될때가 있다. 

회사에서 가장 많이 사용중인 5.7 버전부터 테스트해 본다. 

MySQL 5.7 테스트

mysql> create table tb_test (
    -> col1 int not null primary key,
    -> col2 varchar(10) not null
    -> );
Query OK, 0 rows affected (0.03 sec)

mysql> insert into tb_test values (1, 'aaa');
Query OK, 1 row affected (0.03 sec)

mysql> begin;
Query OK, 0 rows affected (0.00 sec)

mysql> update tb_test set col2 = 'bbb' where col1 = 1;
Query OK, 1 row affected (0.00 sec)
Rows matched: 1  Changed: 1  Warnings: 0

ACTIVE TRANSACTION은 확인되지만 어떤 쿼리인지는 확인이 안된다. 

mysql> show engine innodb status;
...
------------
TRANSACTIONS
------------
Trx id counter 1289
Purge done for trx's n:o < 0 undo n:o < 0 state: running but idle
History list length 0
LIST OF TRANSACTIONS FOR EACH SESSION:
---TRANSACTION 421405712030432, not started
0 lock struct(s), heap size 1136, 0 row lock(s)
---TRANSACTION 1288, ACTIVE 80 sec
2 lock struct(s), heap size 1136, 1 row lock(s), undo log entries 1
MySQL thread id 8, OS thread handle 139930706618112, query id 65 localhost root

mysql> select * from information_schema.innodb_locks;
Empty set, 1 warning (0.00 sec)

mysql> select * from information_schema.innodb_lock_waits;
Empty set, 1 warning (0.00 sec)

mysql> SELECT 
    ->   straight_join
    ->   w.trx_mysql_thread_id waiting_thread,w.trx_id waiting_trx_id,w.trx_query waiting_query,b.trx_mysql_thread_id blocking_thread,
    ->   b.trx_id blocking_trx_id,b.trx_query blocking_query,bl.lock_id blocking_lock_id,bl.lock_mode blocking_lock_mode,
    ->   bl.lock_type blocking_lock_type,bl.lock_table blocking_lock_table,bl.lock_index blocking_lock_index,wl.lock_id waiting_lock_id,
    ->   wl.lock_mode waiting_lock_mode,wl.lock_type waiting_lock_type,wl.lock_table waiting_lock_table,wl.lock_index waiting_lock_index
    -> FROM information_schema.INNODB_LOCK_WAITS ilw
    -> INNER JOIN information_schema.INNODB_TRX b ON b.trx_id = ilw.blocking_trx_id
    -> INNER JOIN information_schema.INNODB_TRX w ON w.trx_id = ilw.requesting_trx_id
    -> INNER JOIN information_schema.INNODB_LOCKS bl ON bl.lock_id = ilw.blocking_lock_id
    -> INNER JOIN information_schema.INNODB_LOCKS wl ON wl.lock_id = ilw.requested_lock_id;
Empty set, 3 warnings (0.00 sec)

다른 세션에서 Lock Wait 상황을 발생시키면?

mysql> begin;
Query OK, 0 rows affected (0.00 sec)

mysql> update tb_test set col2 = 'bbb' where col1 = 1;

Lock Wait 가 발생하면 확인이 된다.

mysql> select * from information_schema.INNODB_LOCK_WAITS;
+-------------------+-------------------+-----------------+------------------+
| requesting_trx_id | requested_lock_id | blocking_trx_id | blocking_lock_id |
+-------------------+-------------------+-----------------+------------------+
| 1289              | 1289:23:3:2       | 1288            | 1288:23:3:2      |
+-------------------+-------------------+-----------------+------------------+
1 row in set, 1 warning (0.00 sec)

mysql> select * from information_schema.INNODB_TRX;
+--------+-----------+---------------------+-----------------------+---------------------+------------+---------------------+------------------------------------------------+---------------------+-------------------+-------------------+------------------+-----------------------+-----------------+-------------------+-------------------------+---------------------+-------------------+------------------------+----------------------------+---------------------------+---------------------------+------------------+----------------------------+
| trx_id | trx_state | trx_started         | trx_requested_lock_id | trx_wait_started    | trx_weight | trx_mysql_thread_id | trx_query                                      | trx_operation_state | trx_tables_in_use | trx_tables_locked | trx_lock_structs | trx_lock_memory_bytes | trx_rows_locked | trx_rows_modified | trx_concurrency_tickets | trx_isolation_level | trx_unique_checks | trx_foreign_key_checks | trx_last_foreign_key_error | trx_adaptive_hash_latched | trx_adaptive_hash_timeout | trx_is_read_only | trx_autocommit_non_locking |
+--------+-----------+---------------------+-----------------------+---------------------+------------+---------------------+------------------------------------------------+---------------------+-------------------+-------------------+------------------+-----------------------+-----------------+-------------------+-------------------------+---------------------+-------------------+------------------------+----------------------------+---------------------------+---------------------------+------------------+----------------------------+
| 1289   | LOCK WAIT | 2022-10-01 15:39:28 | 1289:23:3:2           | 2022-10-01 15:40:35 |          2 |                  10 | update tb_test set col2 = 'bbb' where col1 = 1 | starting index read |                 1 |                 1 |                2 |                  1136 |               2 |                 0 |                       0 | REPEATABLE READ     |                 1 |                      1 | NULL                       |                         0 |                         0 |                0 |                          0 |
| 1288   | RUNNING   | 2022-10-01 15:31:45 | NULL                  | NULL                |          3 |                   8 | NULL                                           | NULL                |                 0 |                 1 |                2 |                  1136 |               1 |                 1 |                       0 | REPEATABLE READ     |                 1 |                      1 | NULL                       |                         0 |                         0 |                0 |                          0 |
+--------+-----------+---------------------+-----------------------+---------------------+------------+---------------------+------------------------------------------------+---------------------+-------------------+-------------------+------------------+-----------------------+-----------------+-------------------+-------------------------+---------------------+-------------------+------------------------+----------------------------+---------------------------+---------------------------+------------------+----------------------------+
2 rows in set (0.00 sec)

mysql> SELECT 
    ->   straight_join
    ->   w.trx_mysql_thread_id waiting_thread,w.trx_id waiting_trx_id,w.trx_query waiting_query,b.trx_mysql_thread_id blocking_thread,
    ->   b.trx_id blocking_trx_id,b.trx_query blocking_query,bl.lock_id blocking_lock_id,bl.lock_mode blocking_lock_mode,
    ->   bl.lock_type blocking_lock_type,bl.lock_table blocking_lock_table,bl.lock_index blocking_lock_index,wl.lock_id waiting_lock_id,
    ->   wl.lock_mode waiting_lock_mode,wl.lock_type waiting_lock_type,wl.lock_table waiting_lock_table,wl.lock_index waiting_lock_index
    -> FROM information_schema.INNODB_LOCK_WAITS ilw
    -> INNER JOIN information_schema.INNODB_TRX b ON b.trx_id = ilw.blocking_trx_id
    -> INNER JOIN information_schema.INNODB_TRX w ON w.trx_id = ilw.requesting_trx_id
    -> INNER JOIN information_schema.INNODB_LOCKS bl ON bl.lock_id = ilw.blocking_lock_id
    -> INNER JOIN information_schema.INNODB_LOCKS wl ON wl.lock_id = ilw.requested_lock_id;
+----------------+----------------+------------------------------------------------+-----------------+-----------------+----------------+------------------+--------------------+--------------------+---------------------+---------------------+-----------------+-------------------+-------------------+--------------------+--------------------+
| waiting_thread | waiting_trx_id | waiting_query                                  | blocking_thread | blocking_trx_id | blocking_query | blocking_lock_id | blocking_lock_mode | blocking_lock_type | blocking_lock_table | blocking_lock_index | waiting_lock_id | waiting_lock_mode | waiting_lock_type | waiting_lock_table | waiting_lock_index |
+----------------+----------------+------------------------------------------------+-----------------+-----------------+----------------+------------------+--------------------+--------------------+---------------------+---------------------+-----------------+-------------------+-------------------+--------------------+--------------------+
|             10 | 1289           | update tb_test set col2 = 'bbb' where col1 = 1 |               8 | 1288            | NULL           | 1288:23:3:2      | X                  | RECORD             | `test`.`tb_test`    | PRIMARY             | 1289:23:3:2     | X                 | RECORD            | `test`.`tb_test`   | PRIMARY            |
+----------------+----------------+------------------------------------------------+-----------------+-----------------+----------------+------------------+--------------------+--------------------+---------------------+---------------------+-----------------+-------------------+-------------------+--------------------+--------------------+
1 row in set, 3 warnings (0.00 sec)

5.7 에서는 Lock Wait 이 발생해야 Lock 정보 확인이 가능하다.

즉 Lock 경합이 없으면 어떤 쿼리가 Lock 잡고 있는지 확인이 안된다. 

아하하하 미친다. (왜 미치냐면 이게 문제가 되고 있어서.)

 

MySQL 8.0 테스트

mysql> create table tb_test (
    -> col1 int not null primary key,
    -> col2 varchar(10) not null
    -> );
Query OK, 0 rows affected (0.03 sec)

mysql> insert into tb_test values (1, 'aaa');
Query OK, 1 row affected (0.03 sec)

mysql> begin;
Query OK, 0 rows affected (0.00 sec)

mysql> update tb_test set col2 = 'bbb' where col1 = 1;
Query OK, 1 row affected (0.00 sec)
Rows matched: 1  Changed: 1  Warnings: 0

ACTIVE TRANSACTION, Lock 쿼리 확인 

mysql> show engine innodb status;
------------
TRANSACTIONS
------------
Trx id counter 7713
Purge done for trx's n:o < 7710 undo n:o < 0 state: running but idle
History list length 0
LIST OF TRANSACTIONS FOR EACH SESSION:
---TRANSACTION 422006973530112, not started
0 lock struct(s), heap size 1128, 0 row lock(s)
---TRANSACTION 422006973528496, not started
0 lock struct(s), heap size 1128, 0 row lock(s)
---TRANSACTION 422006973527688, not started
0 lock struct(s), heap size 1128, 0 row lock(s)
---TRANSACTION 7712, ACTIVE 26 sec
2 lock struct(s), heap size 1128, 1 row lock(s), undo log entries 1
MySQL thread id 9, OS thread handle 140531536729856, query id 26 localhost root

mysql> select * from performance_schema.data_locks;
+--------+----------------------------------------+-----------------------+-----------+----------+---------------+-------------+----------------+-------------------+------------+-----------------------+-----------+---------------+-------------+-----------+
| ENGINE | ENGINE_LOCK_ID                         | ENGINE_TRANSACTION_ID | THREAD_ID | EVENT_ID | OBJECT_SCHEMA | OBJECT_NAME | PARTITION_NAME | SUBPARTITION_NAME | INDEX_NAME | OBJECT_INSTANCE_BEGIN | LOCK_TYPE | LOCK_MODE     | LOCK_STATUS | LOCK_DATA |
+--------+----------------------------------------+-----------------------+-----------+----------+---------------+-------------+----------------+-------------------+------------+-----------------------+-----------+---------------+-------------+-----------+
| INNODB | 140531996818648:1083:140532001090368   |                  7712 |        48 |       32 | test          | tb_test     | NULL           | NULL              | NULL       |       140532001090368 | TABLE     | IX            | GRANTED     | NULL      |
| INNODB | 140531996818648:22:4:2:140532001087456 |                  7712 |        48 |       32 | test          | tb_test     | NULL           | NULL              | PRIMARY    |       140532001087456 | RECORD    | X,REC_NOT_GAP | GRANTED     | 1         |
+--------+----------------------------------------+-----------------------+-----------+----------+---------------+-------------+----------------+-------------------+------------+-----------------------+-----------+---------------+-------------+-----------+
2 rows in set (0.01 sec)

mysql> SELECT straight_join
    ->   dl.thread_id,est.sql_text,dl.object_schema,dl.object_name,dl.index_name,
    ->   dl.lock_type,dl.lock_mode,dl.lock_status,dl.lock_data
    -> FROM performance_schema.data_locks dl 
    -> INNER JOIN performance_schema.events_statements_current est ON dl.thread_id = est.thread_id
    -> ORDER BY est.timer_start,dl.object_instance_begin;
+-----------+------------------------------------------------+---------------+-------------+------------+-----------+---------------+-------------+-----------+
| thread_id | sql_text                                       | object_schema | object_name | index_name | lock_type | lock_mode     | lock_status | lock_data |
+-----------+------------------------------------------------+---------------+-------------+------------+-----------+---------------+-------------+-----------+
|        48 | update tb_test set col2 = 'bbb' where col1 = 1 | test          | tb_test     | PRIMARY    | RECORD    | X,REC_NOT_GAP | GRANTED     | 1         |
|        48 | update tb_test set col2 = 'bbb' where col1 = 1 | test          | tb_test     | NULL       | TABLE     | IX            | GRANTED     | NULL      |
+-----------+------------------------------------------------+---------------+-------------+------------+-----------+---------------+-------------+-----------+
2 rows in set (0.00 sec)

8.0 에서는 Lock Wait 이 발생하지 않아도 Lock 을 잡고 있는 쿼리 확인이 잘된다. 

 

'MySQL' 카테고리의 다른 글

xtrabackup 옵션  (1) 2023.11.30
CPU 사용률 높은 Thread (세션) 확인  (0) 2022.10.05
auto_increment 값 모니터링  (0) 2022.09.28
Query Cache  (0) 2022.09.24
Index Hint  (0) 2022.09.22
SELECT
  t.TABLE_SCHEMA AS `schema`,
  t.TABLE_NAME AS `table`,
  t.AUTO_INCREMENT AS `auto_increment`,
  c.DATA_TYPE AS `pk_type`,
  (t.AUTO_INCREMENT / (CASE DATA_TYPE
                         WHEN 'tinyint' THEN IF(COLUMN_TYPE LIKE '%unsigned', 255, 127)
                         WHEN 'smallint' THEN IF(COLUMN_TYPE LIKE '%unsigned', 65535, 32767)
                         WHEN 'mediumint' THEN IF(COLUMN_TYPE LIKE '%unsigned', 16777215, 8388607)
                         WHEN 'int' THEN IF(COLUMN_TYPE LIKE '%unsigned', 4294967295, 2147483647)
                         WHEN 'bigint' THEN IF(COLUMN_TYPE LIKE '%unsigned', 18446744073709551615, 9223372036854775807)
                         END / 100)) AS `max_value`
FROM information_schema.TABLES t
INNER JOIN information_schema.COLUMNS c ON t.TABLE_SCHEMA = c.TABLE_SCHEMA AND t.TABLE_NAME = c.TABLE_NAME 
WHERE t.AUTO_INCREMENT IS NOT NULL
AND c.COLUMN_KEY = 'PRI'
AND c.DATA_TYPE LIKE '%int';

'MySQL' 카테고리의 다른 글

CPU 사용률 높은 Thread (세션) 확인  (0) 2022.10.05
lock session 찾기  (0) 2022.10.01
Query Cache  (0) 2022.09.24
Index Hint  (0) 2022.09.22
Metadata Lock 조회  (0) 2022.09.22

MySQL 5.7.20 버전부터 deprecate 되었고 8.0 에서는 사라졌다. 

 

쿼리를 Query Cache 에서 조회해보고 Query Cache에 있다면 Parsing, Optimizing, Executing 을 거치지 않고 결과를 반환한다. 

좋지 않은가?

 

근데 왜 없어졌을까?

 

Query Cache 에서는 아래의 두 쿼리가 다르다고 판단한다. 

SELECT * FROM TABLE

select * from table

Query Cache 에서 쿼리가 동일한 것으로 인식되기 위해서는 대소문자, 바이트 까지 동일해야한다. 

쿼리 스트링이 같다고 해도 데이터베이스, 프로토콜버전, 디폴트 문자셋 등이 다른 경우도 다른 쿼리로 판단한다. 

Query Cache 에 저장되어 있는 쿼리 결과의 대상 테이블의 변경 (INSERT, UPDATE, DELETE, ALTER TABLE, TRUNCATE 등등)이 발생할 경우 캐시에서 제거한다. 이때 다른 쓰레드에서 이 데이터를 참조하지 못하도록 Lock을 건다. 

(Query Cache는 여러 세션이 공유하는 부분으로 동기화를 위해 Query Cache Lock을 건다.)

Query Cache Lock 이 풀릴때까지 Query Cache에 접근하는 쓰레드들은 "Waiting for query cache lock" 대기가 걸린다. 

 

테이블 변경이 자주 발생하는 경우 Query Cache 를 사용하는 쿼리들의 대기 시간이 많아진다. 

이런 문제 때문에 사라진듯하다. 

 

'MySQL' 카테고리의 다른 글

lock session 찾기  (0) 2022.10.01
auto_increment 값 모니터링  (0) 2022.09.28
Index Hint  (0) 2022.09.22
Metadata Lock 조회  (0) 2022.09.22
MySQL JSON 사용시 주의사항  (0) 2022.09.16
tbl_name [[AS] alias] [index_hint_list]

index_hint_list:
    index_hint [index_hint] ...

index_hint:
    USE {INDEX|KEY}
      [FOR {JOIN|ORDER BY|GROUP BY}] ([index_list])
  | {IGNORE|FORCE} {INDEX|KEY}
      [FOR {JOIN|ORDER BY|GROUP BY}] (index_list)

index_list:
    index_name [, index_name] ...

 

갑자기 생각이 안날때가 있어서 정리해본다. 

 

- IGNORE INDEX : 특정 인덱스 사용안함

- USE INDEX

  - 옵티마이저에게 지정한 인덱스 사용 권장 

  - 권장이기 때문에 Table Scan 이더 빠르면 Table Scan 을 할수도 있다. 

- FORCE INDEX 

  - 인덱스 사용 강제

  - 인덱스를 사용하지 못하는 경우에만 해당 인덱스를 사용하지 않고 다른 플랜으로 처리 

  - Table Scan 이 더 성능이 좋아도 인덱스 사용 

 

 

 

참고 : https://dev.mysql.com/doc/refman/8.0/en/index-hints.html

'MySQL' 카테고리의 다른 글

auto_increment 값 모니터링  (0) 2022.09.28
Query Cache  (0) 2022.09.24
Metadata Lock 조회  (0) 2022.09.22
MySQL JSON 사용시 주의사항  (0) 2022.09.16
MySQL JSON, Generated Columns  (0) 2022.09.15

mysql 5.7 부터 performance_schema.metadata_locks 가 추가 되었으며 
이를 조회해서 metadata lock 을 잡고 있는 세션 확인이 가능하다.

 

auto_commit = false 상태로 쿼리를 하다보면 본인도 모르게 metadata lock 을 잡을 수 도 있다. 

예를 들면 python 에서 pymysql 로 접속시 기본 설정이 auto_commit = false 로 되어 있다. 

 

metadata lock 조회를 위한 선행 작업 

-- 기본 설정이 YES 지만 확인해본다.
SELECT * FROM performance_schema.setup_consumers WHERE NAME = 'global_instrumentation';

-- ENABLED = YES 가 아닐 경우 업데이트 
UPDATE performance_schema.setup_consumers SET ENABLED = 'YES' WHERE NAME = 'global_instrumentation';

-- MySQL 8.0 에서는 기본설정이 ENABLED = YES 지만 
-- 5.7 에서는 NO 로 되어 있어 확인이 필요하다.
SELECT * FROM performance_schema.setup_instruments WHERE NAME = 'wait/lock/metadata/sql/mdl';

-- ENABLED = YES 가 아닐 경우 업데이트 
UPDATE performance_schema.setup_instruments SET ENABLED = 'YES' WHERE NAME = 'wait/lock/metadata/sql/mdl';

metadata lock 조회

SELECT 
  OBJECT_TYPE, OBJECT_SCHEMA, OBJECT_NAME, LOCK_TYPE, LOCK_STATUS,
  THREAD_ID, PROCESSLIST_ID, PROCESSLIST_INFO
FROM performance_schema.metadata_locks
INNER JOIN performance_schema.threads ON THREAD_ID = OWNER_THREAD_ID
WHERE PROCESSLIST_ID <> CONNECTION_ID();

 

'MySQL' 카테고리의 다른 글

Query Cache  (0) 2022.09.24
Index Hint  (0) 2022.09.22
MySQL JSON 사용시 주의사항  (0) 2022.09.16
MySQL JSON, Generated Columns  (0) 2022.09.15
MySQL Performance Best Practice  (0) 2021.12.26

+ Recent posts